1// Licensed to the Apache Software Foundation (ASF) under one 2// or more contributor license agreements. See the NOTICE file 3// distributed with this work for additional information 4// regarding copyright ownership. The ASF licenses this file 5// to you under the Apache License, Version 2.0 (the 6// "License"); you may not use this file except in compliance 7// with the License. You may obtain a copy of the License at 8// 9// http://www.apache.org/licenses/LICENSE-2.0 10// 11// Unless required by applicable law or agreed to in writing, software 12// distributed under the License is distributed on an "AS IS" BASIS, 13// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14// See the License for the specific language governing permissions and 15// limitations under the License. 16 17// Package arrdata exports arrays and records data ready to be used for tests. 18package arrdata // import "github.com/apache/arrow/go/v6/arrow/internal/arrdata" 19 20import ( 21 "fmt" 22 "sort" 23 24 "github.com/apache/arrow/go/v6/arrow" 25 "github.com/apache/arrow/go/v6/arrow/array" 26 "github.com/apache/arrow/go/v6/arrow/decimal128" 27 "github.com/apache/arrow/go/v6/arrow/float16" 28 "github.com/apache/arrow/go/v6/arrow/internal/testing/types" 29 "github.com/apache/arrow/go/v6/arrow/ipc" 30 "github.com/apache/arrow/go/v6/arrow/memory" 31) 32 33var ( 34 Records = make(map[string][]array.Record) 35 RecordNames []string 36) 37 38func init() { 39 Records["nulls"] = makeNullRecords() 40 Records["primitives"] = makePrimitiveRecords() 41 Records["structs"] = makeStructsRecords() 42 Records["lists"] = makeListsRecords() 43 Records["strings"] = makeStringsRecords() 44 Records["fixed_size_lists"] = makeFixedSizeListsRecords() 45 Records["fixed_width_types"] = makeFixedWidthTypesRecords() 46 Records["fixed_size_binaries"] = makeFixedSizeBinariesRecords() 47 Records["intervals"] = makeIntervalsRecords() 48 Records["durations"] = makeDurationsRecords() 49 Records["decimal128"] = makeDecimal128sRecords() 50 Records["maps"] = makeMapsRecords() 51 Records["extension"] = makeExtensionRecords() 52 53 for k := range Records { 54 RecordNames = append(RecordNames, k) 55 } 56 sort.Strings(RecordNames) 57} 58 59func makeNullRecords() []array.Record { 60 mem := memory.NewGoAllocator() 61 62 meta := arrow.NewMetadata( 63 []string{"k1", "k2", "k3"}, 64 []string{"v1", "v2", "v3"}, 65 ) 66 67 schema := arrow.NewSchema( 68 []arrow.Field{ 69 arrow.Field{Name: "nulls", Type: arrow.Null, Nullable: true}, 70 }, &meta, 71 ) 72 73 mask := []bool{true, false, false, true, true} 74 chunks := [][]array.Interface{ 75 []array.Interface{ 76 arrayOf(mem, []nullT{null, null, null, null, null}, mask), 77 }, 78 []array.Interface{ 79 arrayOf(mem, []nullT{null, null, null, null, null}, mask), 80 }, 81 []array.Interface{ 82 arrayOf(mem, []nullT{null, null, null, null, null}, mask), 83 }, 84 } 85 86 defer func() { 87 for _, chunk := range chunks { 88 for _, col := range chunk { 89 col.Release() 90 } 91 } 92 }() 93 94 recs := make([]array.Record, len(chunks)) 95 for i, chunk := range chunks { 96 recs[i] = array.NewRecord(schema, chunk, -1) 97 } 98 99 return recs 100} 101 102func makePrimitiveRecords() []array.Record { 103 mem := memory.NewGoAllocator() 104 105 meta := arrow.NewMetadata( 106 []string{"k1", "k2", "k3"}, 107 []string{"v1", "v2", "v3"}, 108 ) 109 110 schema := arrow.NewSchema( 111 []arrow.Field{ 112 arrow.Field{Name: "bools", Type: arrow.FixedWidthTypes.Boolean, Nullable: true}, 113 arrow.Field{Name: "int8s", Type: arrow.PrimitiveTypes.Int8, Nullable: true}, 114 arrow.Field{Name: "int16s", Type: arrow.PrimitiveTypes.Int16, Nullable: true}, 115 arrow.Field{Name: "int32s", Type: arrow.PrimitiveTypes.Int32, Nullable: true}, 116 arrow.Field{Name: "int64s", Type: arrow.PrimitiveTypes.Int64, Nullable: true}, 117 arrow.Field{Name: "uint8s", Type: arrow.PrimitiveTypes.Uint8, Nullable: true}, 118 arrow.Field{Name: "uint16s", Type: arrow.PrimitiveTypes.Uint16, Nullable: true}, 119 arrow.Field{Name: "uint32s", Type: arrow.PrimitiveTypes.Uint32, Nullable: true}, 120 arrow.Field{Name: "uint64s", Type: arrow.PrimitiveTypes.Uint64, Nullable: true}, 121 arrow.Field{Name: "float32s", Type: arrow.PrimitiveTypes.Float32, Nullable: true}, 122 arrow.Field{Name: "float64s", Type: arrow.PrimitiveTypes.Float64, Nullable: true}, 123 }, &meta, 124 ) 125 126 mask := []bool{true, false, false, true, true} 127 chunks := [][]array.Interface{ 128 []array.Interface{ 129 arrayOf(mem, []bool{true, false, true, false, true}, mask), 130 arrayOf(mem, []int8{-1, -2, -3, -4, -5}, mask), 131 arrayOf(mem, []int16{-1, -2, -3, -4, -5}, mask), 132 arrayOf(mem, []int32{-1, -2, -3, -4, -5}, mask), 133 arrayOf(mem, []int64{-1, -2, -3, -4, -5}, mask), 134 arrayOf(mem, []uint8{+1, +2, +3, +4, +5}, mask), 135 arrayOf(mem, []uint16{+1, +2, +3, +4, +5}, mask), 136 arrayOf(mem, []uint32{+1, +2, +3, +4, +5}, mask), 137 arrayOf(mem, []uint64{+1, +2, +3, +4, +5}, mask), 138 arrayOf(mem, []float32{+1, +2, +3, +4, +5}, mask), 139 arrayOf(mem, []float64{+1, +2, +3, +4, +5}, mask), 140 }, 141 []array.Interface{ 142 arrayOf(mem, []bool{true, false, true, false, true}, mask), 143 arrayOf(mem, []int8{-11, -12, -13, -14, -15}, mask), 144 arrayOf(mem, []int16{-11, -12, -13, -14, -15}, mask), 145 arrayOf(mem, []int32{-11, -12, -13, -14, -15}, mask), 146 arrayOf(mem, []int64{-11, -12, -13, -14, -15}, mask), 147 arrayOf(mem, []uint8{+11, +12, +13, +14, +15}, mask), 148 arrayOf(mem, []uint16{+11, +12, +13, +14, +15}, mask), 149 arrayOf(mem, []uint32{+11, +12, +13, +14, +15}, mask), 150 arrayOf(mem, []uint64{+11, +12, +13, +14, +15}, mask), 151 arrayOf(mem, []float32{+11, +12, +13, +14, +15}, mask), 152 arrayOf(mem, []float64{+11, +12, +13, +14, +15}, mask), 153 }, 154 []array.Interface{ 155 arrayOf(mem, []bool{true, false, true, false, true}, mask), 156 arrayOf(mem, []int8{-21, -22, -23, -24, -25}, mask), 157 arrayOf(mem, []int16{-21, -22, -23, -24, -25}, mask), 158 arrayOf(mem, []int32{-21, -22, -23, -24, -25}, mask), 159 arrayOf(mem, []int64{-21, -22, -23, -24, -25}, mask), 160 arrayOf(mem, []uint8{+21, +22, +23, +24, +25}, mask), 161 arrayOf(mem, []uint16{+21, +22, +23, +24, +25}, mask), 162 arrayOf(mem, []uint32{+21, +22, +23, +24, +25}, mask), 163 arrayOf(mem, []uint64{+21, +22, +23, +24, +25}, mask), 164 arrayOf(mem, []float32{+21, +22, +23, +24, +25}, mask), 165 arrayOf(mem, []float64{+21, +22, +23, +24, +25}, mask), 166 }, 167 } 168 169 defer func() { 170 for _, chunk := range chunks { 171 for _, col := range chunk { 172 col.Release() 173 } 174 } 175 }() 176 177 recs := make([]array.Record, len(chunks)) 178 for i, chunk := range chunks { 179 recs[i] = array.NewRecord(schema, chunk, -1) 180 } 181 182 return recs 183} 184 185func makeStructsRecords() []array.Record { 186 mem := memory.NewGoAllocator() 187 188 fields := []arrow.Field{ 189 {Name: "f1", Type: arrow.PrimitiveTypes.Int32}, 190 {Name: "f2", Type: arrow.BinaryTypes.String}, 191 } 192 dtype := arrow.StructOf(fields...) 193 schema := arrow.NewSchema([]arrow.Field{{Name: "struct_nullable", Type: dtype, Nullable: true}}, nil) 194 195 mask := []bool{true, false, false, true, true, true, false, true} 196 chunks := [][]array.Interface{ 197 []array.Interface{ 198 structOf(mem, dtype, [][]array.Interface{ 199 []array.Interface{ 200 arrayOf(mem, []int32{-1, -2, -3, -4, -5}, mask[:5]), 201 arrayOf(mem, []string{"111", "222", "333", "444", "555"}, mask[:5]), 202 }, 203 []array.Interface{ 204 arrayOf(mem, []int32{-11, -12, -13, -14, -15}, mask[:5]), 205 arrayOf(mem, []string{"1111", "1222", "1333", "1444", "1555"}, mask[:5]), 206 }, 207 []array.Interface{ 208 arrayOf(mem, []int32{-21, -22, -23, -24, -25}, mask[:5]), 209 arrayOf(mem, []string{"2111", "2222", "2333", "2444", "2555"}, mask[:5]), 210 }, 211 []array.Interface{ 212 arrayOf(mem, []int32{-31, -32, -33, -34, -35}, mask[:5]), 213 arrayOf(mem, []string{"3111", "3222", "3333", "3444", "3555"}, mask[:5]), 214 }, 215 []array.Interface{ 216 arrayOf(mem, []int32{-41, -42, -43, -44, -45}, mask[:5]), 217 arrayOf(mem, []string{"4111", "4222", "4333", "4444", "4555"}, mask[:5]), 218 }, 219 }, []bool{true, false, true, true, true}), 220 }, 221 []array.Interface{ 222 structOf(mem, dtype, [][]array.Interface{ 223 []array.Interface{ 224 arrayOf(mem, []int32{1, 2, 3, 4, 5}, mask[:5]), 225 arrayOf(mem, []string{"-111", "-222", "-333", "-444", "-555"}, mask[:5]), 226 }, 227 []array.Interface{ 228 arrayOf(mem, []int32{11, 12, 13, 14, 15}, mask[:5]), 229 arrayOf(mem, []string{"-1111", "-1222", "-1333", "-1444", "-1555"}, mask[:5]), 230 }, 231 []array.Interface{ 232 arrayOf(mem, []int32{21, 22, 23, 24, 25}, mask[:5]), 233 arrayOf(mem, []string{"-2111", "-2222", "-2333", "-2444", "-2555"}, mask[:5]), 234 }, 235 []array.Interface{ 236 arrayOf(mem, []int32{31, 32, 33, 34, 35}, mask[:5]), 237 arrayOf(mem, []string{"-3111", "-3222", "-3333", "-3444", "-3555"}, mask[:5]), 238 }, 239 []array.Interface{ 240 arrayOf(mem, []int32{41, 42, 43, 44, 45}, mask[:5]), 241 arrayOf(mem, []string{"-4111", "-4222", "-4333", "-4444", "-4555"}, mask[:5]), 242 }, 243 }, []bool{true, false, false, true, true}), 244 }, 245 } 246 247 defer func() { 248 for _, chunk := range chunks { 249 for _, col := range chunk { 250 col.Release() 251 } 252 } 253 }() 254 255 recs := make([]array.Record, len(chunks)) 256 for i, chunk := range chunks { 257 recs[i] = array.NewRecord(schema, chunk, -1) 258 } 259 260 return recs 261} 262 263func makeListsRecords() []array.Record { 264 mem := memory.NewGoAllocator() 265 dtype := arrow.ListOf(arrow.PrimitiveTypes.Int32) 266 schema := arrow.NewSchema([]arrow.Field{ 267 {Name: "list_nullable", Type: dtype, Nullable: true}, 268 }, nil) 269 270 mask := []bool{true, false, false, true, true} 271 272 chunks := [][]array.Interface{ 273 []array.Interface{ 274 listOf(mem, []array.Interface{ 275 arrayOf(mem, []int32{1, 2, 3, 4, 5}, mask), 276 arrayOf(mem, []int32{11, 12, 13, 14, 15}, mask), 277 arrayOf(mem, []int32{21, 22, 23, 24, 25}, mask), 278 }, nil), 279 }, 280 []array.Interface{ 281 listOf(mem, []array.Interface{ 282 arrayOf(mem, []int32{-1, -2, -3, -4, -5}, mask), 283 arrayOf(mem, []int32{-11, -12, -13, -14, -15}, mask), 284 arrayOf(mem, []int32{-21, -22, -23, -24, -25}, mask), 285 }, nil), 286 }, 287 []array.Interface{ 288 listOf(mem, []array.Interface{ 289 arrayOf(mem, []int32{-1, -2, -3, -4, -5}, mask), 290 arrayOf(mem, []int32{-11, -12, -13, -14, -15}, mask), 291 arrayOf(mem, []int32{-21, -22, -23, -24, -25}, mask), 292 }, []bool{true, false, true}), 293 }, 294 []array.Interface{ 295 func() array.Interface { 296 bldr := array.NewListBuilder(mem, arrow.PrimitiveTypes.Int32) 297 defer bldr.Release() 298 299 return bldr.NewListArray() 300 }(), 301 }, 302 } 303 304 defer func() { 305 for _, chunk := range chunks { 306 for _, col := range chunk { 307 col.Release() 308 } 309 } 310 }() 311 312 recs := make([]array.Record, len(chunks)) 313 for i, chunk := range chunks { 314 recs[i] = array.NewRecord(schema, chunk, -1) 315 } 316 317 return recs 318} 319 320func makeFixedSizeListsRecords() []array.Record { 321 mem := memory.NewGoAllocator() 322 const N = 3 323 dtype := arrow.FixedSizeListOf(N, arrow.PrimitiveTypes.Int32) 324 schema := arrow.NewSchema([]arrow.Field{ 325 {Name: "fixed_size_list_nullable", Type: dtype, Nullable: true}, 326 }, nil) 327 328 mask := []bool{true, false, true} 329 330 chunks := [][]array.Interface{ 331 []array.Interface{ 332 fixedSizeListOf(mem, N, []array.Interface{ 333 arrayOf(mem, []int32{1, 2, 3}, mask), 334 arrayOf(mem, []int32{11, 12, 13}, mask), 335 arrayOf(mem, []int32{21, 22, 23}, mask), 336 }, nil), 337 }, 338 []array.Interface{ 339 fixedSizeListOf(mem, N, []array.Interface{ 340 arrayOf(mem, []int32{-1, -2, -3}, mask), 341 arrayOf(mem, []int32{-11, -12, -13}, mask), 342 arrayOf(mem, []int32{-21, -22, -23}, mask), 343 }, nil), 344 }, 345 []array.Interface{ 346 fixedSizeListOf(mem, N, []array.Interface{ 347 arrayOf(mem, []int32{-1, -2, -3}, mask), 348 arrayOf(mem, []int32{-11, -12, -13}, mask), 349 arrayOf(mem, []int32{-21, -22, -23}, mask), 350 }, []bool{true, false, true}), 351 }, 352 } 353 354 defer func() { 355 for _, chunk := range chunks { 356 for _, col := range chunk { 357 col.Release() 358 } 359 } 360 }() 361 362 recs := make([]array.Record, len(chunks)) 363 for i, chunk := range chunks { 364 recs[i] = array.NewRecord(schema, chunk, -1) 365 } 366 367 return recs 368} 369 370func makeStringsRecords() []array.Record { 371 mem := memory.NewGoAllocator() 372 schema := arrow.NewSchema([]arrow.Field{ 373 {Name: "strings", Type: arrow.BinaryTypes.String}, 374 {Name: "bytes", Type: arrow.BinaryTypes.Binary}, 375 }, nil) 376 377 mask := []bool{true, false, false, true, true} 378 chunks := [][]array.Interface{ 379 []array.Interface{ 380 arrayOf(mem, []string{"1é", "2", "3", "4", "5"}, mask), 381 arrayOf(mem, [][]byte{[]byte("1é"), []byte("2"), []byte("3"), []byte("4"), []byte("5")}, mask), 382 }, 383 []array.Interface{ 384 arrayOf(mem, []string{"11", "22", "33", "44", "55"}, mask), 385 arrayOf(mem, [][]byte{[]byte("11"), []byte("22"), []byte("33"), []byte("44"), []byte("55")}, mask), 386 }, 387 []array.Interface{ 388 arrayOf(mem, []string{"111", "222", "333", "444", "555"}, mask), 389 arrayOf(mem, [][]byte{[]byte("111"), []byte("222"), []byte("333"), []byte("444"), []byte("555")}, mask), 390 }, 391 } 392 393 defer func() { 394 for _, chunk := range chunks { 395 for _, col := range chunk { 396 col.Release() 397 } 398 } 399 }() 400 401 recs := make([]array.Record, len(chunks)) 402 for i, chunk := range chunks { 403 recs[i] = array.NewRecord(schema, chunk, -1) 404 } 405 406 return recs 407} 408 409type ( 410 nullT struct{} 411 time32s arrow.Time32 412 time32ms arrow.Time32 413 time64ns arrow.Time64 414 time64us arrow.Time64 415 timestamp_s arrow.Timestamp 416 timestamp_ms arrow.Timestamp 417 timestamp_us arrow.Timestamp 418 timestamp_ns arrow.Timestamp 419) 420 421var ( 422 null nullT 423) 424 425func makeFixedWidthTypesRecords() []array.Record { 426 mem := memory.NewGoAllocator() 427 schema := arrow.NewSchema( 428 []arrow.Field{ 429 arrow.Field{Name: "float16s", Type: arrow.FixedWidthTypes.Float16, Nullable: true}, 430 arrow.Field{Name: "time32ms", Type: arrow.FixedWidthTypes.Time32ms, Nullable: true}, 431 arrow.Field{Name: "time32s", Type: arrow.FixedWidthTypes.Time32s, Nullable: true}, 432 arrow.Field{Name: "time64ns", Type: arrow.FixedWidthTypes.Time64ns, Nullable: true}, 433 arrow.Field{Name: "time64us", Type: arrow.FixedWidthTypes.Time64us, Nullable: true}, 434 arrow.Field{Name: "timestamp_s", Type: arrow.FixedWidthTypes.Timestamp_s, Nullable: true}, 435 arrow.Field{Name: "timestamp_ms", Type: arrow.FixedWidthTypes.Timestamp_ms, Nullable: true}, 436 arrow.Field{Name: "timestamp_us", Type: arrow.FixedWidthTypes.Timestamp_us, Nullable: true}, 437 arrow.Field{Name: "timestamp_ns", Type: arrow.FixedWidthTypes.Timestamp_ns, Nullable: true}, 438 arrow.Field{Name: "date32s", Type: arrow.FixedWidthTypes.Date32, Nullable: true}, 439 arrow.Field{Name: "date64s", Type: arrow.FixedWidthTypes.Date64, Nullable: true}, 440 }, nil, 441 ) 442 443 float16s := func(vs []float32) []float16.Num { 444 o := make([]float16.Num, len(vs)) 445 for i, v := range vs { 446 o[i] = float16.New(v) 447 } 448 return o 449 } 450 451 mask := []bool{true, false, false, true, true} 452 chunks := [][]array.Interface{ 453 []array.Interface{ 454 arrayOf(mem, float16s([]float32{+1, +2, +3, +4, +5}), mask), 455 arrayOf(mem, []time32ms{-2, -1, 0, +1, +2}, mask), 456 arrayOf(mem, []time32s{-2, -1, 0, +1, +2}, mask), 457 arrayOf(mem, []time64ns{-2, -1, 0, +1, +2}, mask), 458 arrayOf(mem, []time64us{-2, -1, 0, +1, +2}, mask), 459 arrayOf(mem, []timestamp_s{0, +1, +2, +3, +4}, mask), 460 arrayOf(mem, []timestamp_ms{0, +1, +2, +3, +4}, mask), 461 arrayOf(mem, []timestamp_us{0, +1, +2, +3, +4}, mask), 462 arrayOf(mem, []timestamp_ns{0, +1, +2, +3, +4}, mask), 463 arrayOf(mem, []arrow.Date32{-2, -1, 0, +1, +2}, mask), 464 arrayOf(mem, []arrow.Date64{-2, -1, 0, +1, +2}, mask), 465 }, 466 []array.Interface{ 467 arrayOf(mem, float16s([]float32{+11, +12, +13, +14, +15}), mask), 468 arrayOf(mem, []time32ms{-12, -11, 10, +11, +12}, mask), 469 arrayOf(mem, []time32s{-12, -11, 10, +11, +12}, mask), 470 arrayOf(mem, []time64ns{-12, -11, 10, +11, +12}, mask), 471 arrayOf(mem, []time64us{-12, -11, 10, +11, +12}, mask), 472 arrayOf(mem, []timestamp_s{10, +11, +12, +13, +14}, mask), 473 arrayOf(mem, []timestamp_ms{10, +11, +12, +13, +14}, mask), 474 arrayOf(mem, []timestamp_us{10, +11, +12, +13, +14}, mask), 475 arrayOf(mem, []timestamp_ns{10, +11, +12, +13, +14}, mask), 476 arrayOf(mem, []arrow.Date32{-12, -11, 10, +11, +12}, mask), 477 arrayOf(mem, []arrow.Date64{-12, -11, 10, +11, +12}, mask), 478 }, 479 []array.Interface{ 480 arrayOf(mem, float16s([]float32{+21, +22, +23, +24, +25}), mask), 481 arrayOf(mem, []time32ms{-22, -21, 20, +21, +22}, mask), 482 arrayOf(mem, []time32s{-22, -21, 20, +21, +22}, mask), 483 arrayOf(mem, []time64ns{-22, -21, 20, +21, +22}, mask), 484 arrayOf(mem, []time64us{-22, -21, 20, +21, +22}, mask), 485 arrayOf(mem, []timestamp_s{20, +21, +22, +23, +24}, mask), 486 arrayOf(mem, []timestamp_ms{20, +21, +22, +23, +24}, mask), 487 arrayOf(mem, []timestamp_us{20, +21, +22, +23, +24}, mask), 488 arrayOf(mem, []timestamp_ns{20, +21, +22, +23, +24}, mask), 489 arrayOf(mem, []arrow.Date32{-22, -21, 20, +21, +22}, mask), 490 arrayOf(mem, []arrow.Date64{-22, -21, 20, +21, +22}, mask), 491 }, 492 } 493 494 defer func() { 495 for _, chunk := range chunks { 496 for _, col := range chunk { 497 col.Release() 498 } 499 } 500 }() 501 502 recs := make([]array.Record, len(chunks)) 503 for i, chunk := range chunks { 504 recs[i] = array.NewRecord(schema, chunk, -1) 505 } 506 507 return recs 508} 509 510type fsb3 string 511 512func makeFixedSizeBinariesRecords() []array.Record { 513 mem := memory.NewGoAllocator() 514 schema := arrow.NewSchema( 515 []arrow.Field{ 516 arrow.Field{Name: "fixed_size_binary_3", Type: &arrow.FixedSizeBinaryType{ByteWidth: 3}, Nullable: true}, 517 }, nil, 518 ) 519 520 mask := []bool{true, false, false, true, true} 521 chunks := [][]array.Interface{ 522 []array.Interface{ 523 arrayOf(mem, []fsb3{"001", "002", "003", "004", "005"}, mask), 524 }, 525 []array.Interface{ 526 arrayOf(mem, []fsb3{"011", "012", "013", "014", "015"}, mask), 527 }, 528 []array.Interface{ 529 arrayOf(mem, []fsb3{"021", "022", "023", "024", "025"}, mask), 530 }, 531 } 532 533 defer func() { 534 for _, chunk := range chunks { 535 for _, col := range chunk { 536 col.Release() 537 } 538 } 539 }() 540 541 recs := make([]array.Record, len(chunks)) 542 for i, chunk := range chunks { 543 recs[i] = array.NewRecord(schema, chunk, -1) 544 } 545 546 return recs 547} 548 549func makeIntervalsRecords() []array.Record { 550 mem := memory.NewGoAllocator() 551 552 schema := arrow.NewSchema( 553 []arrow.Field{ 554 arrow.Field{Name: "months", Type: arrow.FixedWidthTypes.MonthInterval, Nullable: true}, 555 arrow.Field{Name: "days", Type: arrow.FixedWidthTypes.DayTimeInterval, Nullable: true}, 556 arrow.Field{Name: "nanos", Type: arrow.FixedWidthTypes.MonthDayNanoInterval, Nullable: true}, 557 }, nil, 558 ) 559 560 mask := []bool{true, false, false, true, true} 561 chunks := [][]array.Interface{ 562 []array.Interface{ 563 arrayOf(mem, []arrow.MonthInterval{1, 2, 3, 4, 5}, mask), 564 arrayOf(mem, []arrow.DayTimeInterval{{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}, mask), 565 arrayOf(mem, []arrow.MonthDayNanoInterval{{1, 1, 1000}, {2, 2, 2000}, {3, 3, 3000}, {4, 4, 4000}, {5, 5, 5000}}, mask), 566 }, 567 []array.Interface{ 568 arrayOf(mem, []arrow.MonthInterval{-11, -12, -13, -14, -15}, mask), 569 arrayOf(mem, []arrow.DayTimeInterval{{-11, -11}, {-12, -12}, {-13, -13}, {-14, -14}, {-15, -15}}, mask), 570 arrayOf(mem, []arrow.MonthDayNanoInterval{{-11, -11, -11000}, {-12, -12, -12000}, {-13, -13, -13000}, {-14, -14, -14000}, {-15, -15, -15000}}, mask), 571 }, 572 []array.Interface{ 573 arrayOf(mem, []arrow.MonthInterval{21, 22, 23, 24, 25, 0}, append(mask, true)), 574 arrayOf(mem, []arrow.DayTimeInterval{{21, 21}, {22, 22}, {23, 23}, {24, 24}, {25, 25}, {0, 0}}, append(mask, true)), 575 arrayOf(mem, []arrow.MonthDayNanoInterval{{21, 21, 21000}, {22, 22, 22000}, {23, 23, 23000}, {24, 24, 24000}, {25, 25, 25000}, {0, 0, 0}}, append(mask, true)), 576 }, 577 } 578 579 defer func() { 580 for _, chunk := range chunks { 581 for _, col := range chunk { 582 col.Release() 583 } 584 } 585 }() 586 587 recs := make([]array.Record, len(chunks)) 588 for i, chunk := range chunks { 589 recs[i] = array.NewRecord(schema, chunk, -1) 590 } 591 592 return recs 593} 594 595type ( 596 duration_s arrow.Duration 597 duration_ms arrow.Duration 598 duration_us arrow.Duration 599 duration_ns arrow.Duration 600) 601 602func makeDurationsRecords() []array.Record { 603 mem := memory.NewGoAllocator() 604 605 schema := arrow.NewSchema( 606 []arrow.Field{ 607 arrow.Field{Name: "durations-s", Type: &arrow.DurationType{Unit: arrow.Second}, Nullable: true}, 608 arrow.Field{Name: "durations-ms", Type: &arrow.DurationType{Unit: arrow.Millisecond}, Nullable: true}, 609 arrow.Field{Name: "durations-us", Type: &arrow.DurationType{Unit: arrow.Microsecond}, Nullable: true}, 610 arrow.Field{Name: "durations-ns", Type: &arrow.DurationType{Unit: arrow.Nanosecond}, Nullable: true}, 611 }, nil, 612 ) 613 614 mask := []bool{true, false, false, true, true} 615 chunks := [][]array.Interface{ 616 []array.Interface{ 617 arrayOf(mem, []duration_s{1, 2, 3, 4, 5}, mask), 618 arrayOf(mem, []duration_ms{1, 2, 3, 4, 5}, mask), 619 arrayOf(mem, []duration_us{1, 2, 3, 4, 5}, mask), 620 arrayOf(mem, []duration_ns{1, 2, 3, 4, 5}, mask), 621 }, 622 []array.Interface{ 623 arrayOf(mem, []duration_s{11, 12, 13, 14, 15}, mask), 624 arrayOf(mem, []duration_ms{11, 12, 13, 14, 15}, mask), 625 arrayOf(mem, []duration_us{11, 12, 13, 14, 15}, mask), 626 arrayOf(mem, []duration_ns{11, 12, 13, 14, 15}, mask), 627 }, 628 []array.Interface{ 629 arrayOf(mem, []duration_s{21, 22, 23, 24, 25}, mask), 630 arrayOf(mem, []duration_ms{21, 22, 23, 24, 25}, mask), 631 arrayOf(mem, []duration_us{21, 22, 23, 24, 25}, mask), 632 arrayOf(mem, []duration_ns{21, 22, 23, 24, 25}, mask), 633 }, 634 } 635 636 defer func() { 637 for _, chunk := range chunks { 638 for _, col := range chunk { 639 col.Release() 640 } 641 } 642 }() 643 644 recs := make([]array.Record, len(chunks)) 645 for i, chunk := range chunks { 646 recs[i] = array.NewRecord(schema, chunk, -1) 647 } 648 649 return recs 650} 651 652var ( 653 decimal128Type = &arrow.Decimal128Type{Precision: 10, Scale: 1} 654) 655 656func makeDecimal128sRecords() []array.Record { 657 mem := memory.NewGoAllocator() 658 schema := arrow.NewSchema( 659 []arrow.Field{ 660 arrow.Field{Name: "dec128s", Type: decimal128Type, Nullable: true}, 661 }, nil, 662 ) 663 664 dec128s := func(vs []int64) []decimal128.Num { 665 o := make([]decimal128.Num, len(vs)) 666 for i, v := range vs { 667 o[i] = decimal128.New(v, uint64(v)) 668 } 669 return o 670 } 671 672 mask := []bool{true, false, false, true, true} 673 chunks := [][]array.Interface{ 674 []array.Interface{ 675 arrayOf(mem, dec128s([]int64{31, 32, 33, 34, 35}), mask), 676 }, 677 []array.Interface{ 678 arrayOf(mem, dec128s([]int64{41, 42, 43, 44, 45}), mask), 679 }, 680 []array.Interface{ 681 arrayOf(mem, dec128s([]int64{51, 52, 53, 54, 55}), mask), 682 }, 683 } 684 685 defer func() { 686 for _, chunk := range chunks { 687 for _, col := range chunk { 688 col.Release() 689 } 690 } 691 }() 692 693 recs := make([]array.Record, len(chunks)) 694 for i, chunk := range chunks { 695 recs[i] = array.NewRecord(schema, chunk, -1) 696 } 697 698 return recs 699} 700 701func makeMapsRecords() []array.Record { 702 mem := memory.NewGoAllocator() 703 dtype := arrow.MapOf(arrow.PrimitiveTypes.Int32, arrow.BinaryTypes.String) 704 dtype.KeysSorted = true 705 schema := arrow.NewSchema([]arrow.Field{{Name: "map_int_utf8", Type: dtype, Nullable: true}}, nil) 706 707 mask := []bool{true, false, false, true, true} 708 chunks := [][]array.Interface{ 709 { 710 mapOf(mem, dtype.KeysSorted, []array.Interface{ 711 structOf(mem, dtype.ValueType(), [][]array.Interface{ 712 { 713 arrayOf(mem, []int32{-1, -2, -3, -4, -5}, nil), 714 arrayOf(mem, []string{"111", "222", "333", "444", "555"}, mask[:5]), 715 }, 716 { 717 arrayOf(mem, []int32{-1, -2, -3, -4, -5}, nil), 718 arrayOf(mem, []string{"1111", "1222", "1333", "1444", "1555"}, mask[:5]), 719 }, 720 { 721 arrayOf(mem, []int32{-1, -2, -3, -4, -5}, nil), 722 arrayOf(mem, []string{"2111", "2222", "2333", "2444", "2555"}, mask[:5]), 723 }, 724 { 725 arrayOf(mem, []int32{-1, -2, -3, -4, -5}, nil), 726 arrayOf(mem, []string{"3111", "3222", "3333", "3444", "3555"}, mask[:5]), 727 }, 728 { 729 arrayOf(mem, []int32{-1, -2, -3, -4, -5}, nil), 730 arrayOf(mem, []string{"4111", "4222", "4333", "4444", "4555"}, mask[:5]), 731 }, 732 }, nil), 733 structOf(mem, dtype.ValueType(), [][]array.Interface{ 734 { 735 arrayOf(mem, []int32{1, 2, 3, 4, 5}, nil), 736 arrayOf(mem, []string{"-111", "-222", "-333", "-444", "-555"}, mask[:5]), 737 }, 738 { 739 arrayOf(mem, []int32{1, 2, 3, 4, 5}, nil), 740 arrayOf(mem, []string{"-1111", "-1222", "-1333", "-1444", "-1555"}, mask[:5]), 741 }, 742 { 743 arrayOf(mem, []int32{1, 2, 3, 4, 5}, nil), 744 arrayOf(mem, []string{"-2111", "-2222", "-2333", "-2444", "-2555"}, mask[:5]), 745 }, 746 { 747 arrayOf(mem, []int32{1, 2, 3, 4, 5}, nil), 748 arrayOf(mem, []string{"-3111", "-3222", "-3333", "-3444", "-3555"}, mask[:5]), 749 }, 750 { 751 arrayOf(mem, []int32{1, 2, 3, 4, 5}, nil), 752 arrayOf(mem, []string{"-4111", "-4222", "-4333", "-4444", "-4555"}, mask[:5]), 753 }, 754 }, nil), 755 }, []bool{true, false, true, true, true}), 756 }, 757 { 758 mapOf(mem, dtype.KeysSorted, []array.Interface{ 759 structOf(mem, dtype.ValueType(), [][]array.Interface{ 760 { 761 arrayOf(mem, []int32{1, 2, 3, 4, 5}, nil), 762 arrayOf(mem, []string{"-111", "-222", "-333", "-444", "-555"}, mask[:5]), 763 }, 764 { 765 arrayOf(mem, []int32{1, 2, 3, 4, 5}, nil), 766 arrayOf(mem, []string{"-1111", "-1222", "-1333", "-1444", "-1555"}, mask[:5]), 767 }, 768 { 769 arrayOf(mem, []int32{1, 2, 3, 4, 5}, nil), 770 arrayOf(mem, []string{"-2111", "-2222", "-2333", "-2444", "-2555"}, mask[:5]), 771 }, 772 { 773 arrayOf(mem, []int32{1, 2, 3, 4, 5}, nil), 774 arrayOf(mem, []string{"-3111", "-3222", "-3333", "-3444", "-3555"}, mask[:5]), 775 }, 776 { 777 arrayOf(mem, []int32{1, 2, 3, 4, 5}, nil), 778 arrayOf(mem, []string{"-4111", "-4222", "-4333", "-4444", "-4555"}, mask[:5]), 779 }, 780 }, nil), 781 structOf(mem, dtype.ValueType(), [][]array.Interface{ 782 { 783 arrayOf(mem, []int32{-1, -2, -3, -4, -5}, nil), 784 arrayOf(mem, []string{"111", "222", "333", "444", "555"}, mask[:5]), 785 }, 786 { 787 arrayOf(mem, []int32{-1, -2, -3, -4, -5}, nil), 788 arrayOf(mem, []string{"1111", "1222", "1333", "1444", "1555"}, mask[:5]), 789 }, 790 { 791 arrayOf(mem, []int32{-1, -2, -3, -4, -5}, nil), 792 arrayOf(mem, []string{"2111", "2222", "2333", "2444", "2555"}, mask[:5]), 793 }, 794 { 795 arrayOf(mem, []int32{-1, -2, -3, -4, -5}, nil), 796 arrayOf(mem, []string{"3111", "3222", "3333", "3444", "3555"}, mask[:5]), 797 }, 798 { 799 arrayOf(mem, []int32{-1, -2, -3, -4, -5}, nil), 800 arrayOf(mem, []string{"4111", "4222", "4333", "4444", "4555"}, mask[:5]), 801 }, 802 }, nil), 803 }, []bool{true, false, true, true, true}), 804 }, 805 } 806 807 defer func() { 808 for _, chunk := range chunks { 809 for _, col := range chunk { 810 col.Release() 811 } 812 } 813 }() 814 815 recs := make([]array.Record, len(chunks)) 816 for i, chunk := range chunks { 817 recs[i] = array.NewRecord(schema, chunk, -1) 818 } 819 820 return recs 821} 822 823func makeExtensionRecords() []array.Record { 824 mem := memory.NewGoAllocator() 825 826 p1Type := types.NewParametric1Type(6) 827 p2Type := types.NewParametric1Type(12) 828 p3Type := types.NewParametric2Type(2) 829 p4Type := types.NewParametric2Type(3) 830 p5Type := types.NewExtStructType() 831 832 arrow.RegisterExtensionType(p1Type) 833 arrow.RegisterExtensionType(p3Type) 834 arrow.RegisterExtensionType(p4Type) 835 arrow.RegisterExtensionType(p5Type) 836 837 meta := arrow.NewMetadata( 838 []string{"k1", "k2"}, 839 []string{"v1", "v2"}, 840 ) 841 842 unregisteredMeta := arrow.NewMetadata( 843 append(meta.Keys(), ipc.ExtensionTypeKeyName, ipc.ExtensionMetadataKeyName), 844 append(meta.Values(), "unregistered", "")) 845 846 schema := arrow.NewSchema( 847 []arrow.Field{ 848 {Name: "p1", Type: p1Type, Nullable: true, Metadata: meta}, 849 {Name: "p2", Type: p2Type, Nullable: true, Metadata: meta}, 850 {Name: "p3", Type: p3Type, Nullable: true, Metadata: meta}, 851 {Name: "p4", Type: p4Type, Nullable: true, Metadata: meta}, 852 {Name: "p5", Type: p5Type, Nullable: true, Metadata: meta}, 853 {Name: "unreg", Type: arrow.PrimitiveTypes.Int8, Nullable: true, Metadata: unregisteredMeta}, 854 }, nil) 855 856 mask := []bool{true, false, true, true, false} 857 chunks := [][]array.Interface{ 858 { 859 extArray(mem, p1Type, []int32{1, -1, 2, 3, -1}, mask), 860 extArray(mem, p2Type, []int32{2, -1, 3, 4, -1}, mask), 861 extArray(mem, p3Type, []int32{5, -1, 6, 7, 8}, mask), 862 extArray(mem, p4Type, []int32{5, -1, 7, 9, -1}, mask), 863 extArray(mem, p5Type, [][]array.Interface{ 864 { 865 arrayOf(mem, []int64{1, -1, 2, 3, -1}, mask), 866 arrayOf(mem, []float64{0.1, -1, 0.2, 0.3, -1}, mask), 867 }, 868 }, mask), 869 arrayOf(mem, []int8{-1, -2, -3, -4, -5}, mask), 870 }, 871 { 872 extArray(mem, p1Type, []int32{10, -1, 20, 30, -1}, mask), 873 extArray(mem, p2Type, []int32{20, -1, 30, 40, -1}, mask), 874 extArray(mem, p3Type, []int32{50, -1, 60, 70, 8}, mask), 875 extArray(mem, p4Type, []int32{50, -1, 70, 90, -1}, mask), 876 extArray(mem, p5Type, [][]array.Interface{ 877 { 878 arrayOf(mem, []int64{10, -1, 20, 30, -1}, mask), 879 arrayOf(mem, []float64{0.01, -1, 0.02, 0.03, -1}, mask), 880 }, 881 }, mask), 882 arrayOf(mem, []int8{-11, -12, -13, -14, -15}, mask), 883 }, 884 } 885 886 defer func() { 887 for _, chunk := range chunks { 888 for _, col := range chunk { 889 col.Release() 890 } 891 } 892 }() 893 894 recs := make([]array.Record, len(chunks)) 895 for i, chunk := range chunks { 896 recs[i] = array.NewRecord(schema, chunk, -1) 897 } 898 899 return recs 900} 901 902func extArray(mem memory.Allocator, dt arrow.ExtensionType, a interface{}, valids []bool) array.Interface { 903 var storage array.Interface 904 switch st := dt.StorageType().(type) { 905 case *arrow.StructType: 906 storage = structOf(mem, st, a.([][]array.Interface), valids) 907 case *arrow.MapType: 908 storage = mapOf(mem, false, a.([]array.Interface), valids) 909 case *arrow.ListType: 910 storage = listOf(mem, a.([]array.Interface), valids) 911 default: 912 storage = arrayOf(mem, a, valids) 913 } 914 defer storage.Release() 915 916 return array.NewExtensionArrayWithStorage(dt, storage) 917} 918 919func arrayOf(mem memory.Allocator, a interface{}, valids []bool) array.Interface { 920 if mem == nil { 921 mem = memory.NewGoAllocator() 922 } 923 924 switch a := a.(type) { 925 case []nullT: 926 return array.NewNull(len(a)) 927 928 case []bool: 929 bldr := array.NewBooleanBuilder(mem) 930 defer bldr.Release() 931 932 bldr.AppendValues(a, valids) 933 return bldr.NewBooleanArray() 934 935 case []int8: 936 bldr := array.NewInt8Builder(mem) 937 defer bldr.Release() 938 939 bldr.AppendValues(a, valids) 940 return bldr.NewInt8Array() 941 942 case []int16: 943 bldr := array.NewInt16Builder(mem) 944 defer bldr.Release() 945 946 bldr.AppendValues(a, valids) 947 return bldr.NewInt16Array() 948 949 case []int32: 950 bldr := array.NewInt32Builder(mem) 951 defer bldr.Release() 952 953 bldr.AppendValues(a, valids) 954 return bldr.NewInt32Array() 955 956 case []int64: 957 bldr := array.NewInt64Builder(mem) 958 defer bldr.Release() 959 960 bldr.AppendValues(a, valids) 961 return bldr.NewInt64Array() 962 963 case []uint8: 964 bldr := array.NewUint8Builder(mem) 965 defer bldr.Release() 966 967 bldr.AppendValues(a, valids) 968 return bldr.NewUint8Array() 969 970 case []uint16: 971 bldr := array.NewUint16Builder(mem) 972 defer bldr.Release() 973 974 bldr.AppendValues(a, valids) 975 return bldr.NewUint16Array() 976 977 case []uint32: 978 bldr := array.NewUint32Builder(mem) 979 defer bldr.Release() 980 981 bldr.AppendValues(a, valids) 982 return bldr.NewUint32Array() 983 984 case []uint64: 985 bldr := array.NewUint64Builder(mem) 986 defer bldr.Release() 987 988 bldr.AppendValues(a, valids) 989 return bldr.NewUint64Array() 990 991 case []float16.Num: 992 bldr := array.NewFloat16Builder(mem) 993 defer bldr.Release() 994 995 bldr.AppendValues(a, valids) 996 return bldr.NewFloat16Array() 997 998 case []float32: 999 bldr := array.NewFloat32Builder(mem) 1000 defer bldr.Release() 1001 1002 bldr.AppendValues(a, valids) 1003 return bldr.NewFloat32Array() 1004 1005 case []float64: 1006 bldr := array.NewFloat64Builder(mem) 1007 defer bldr.Release() 1008 1009 bldr.AppendValues(a, valids) 1010 return bldr.NewFloat64Array() 1011 1012 case []decimal128.Num: 1013 bldr := array.NewDecimal128Builder(mem, decimal128Type) 1014 defer bldr.Release() 1015 1016 bldr.AppendValues(a, valids) 1017 aa := bldr.NewDecimal128Array() 1018 return aa 1019 1020 case []string: 1021 bldr := array.NewStringBuilder(mem) 1022 defer bldr.Release() 1023 1024 bldr.AppendValues(a, valids) 1025 return bldr.NewStringArray() 1026 1027 case [][]byte: 1028 bldr := array.NewBinaryBuilder(mem, arrow.BinaryTypes.Binary) 1029 defer bldr.Release() 1030 1031 bldr.AppendValues(a, valids) 1032 return bldr.NewBinaryArray() 1033 1034 case []time32s: 1035 bldr := array.NewTime32Builder(mem, arrow.FixedWidthTypes.Time32s.(*arrow.Time32Type)) 1036 defer bldr.Release() 1037 1038 vs := make([]arrow.Time32, len(a)) 1039 for i, v := range a { 1040 vs[i] = arrow.Time32(v) 1041 } 1042 bldr.AppendValues(vs, valids) 1043 return bldr.NewArray() 1044 1045 case []time32ms: 1046 bldr := array.NewTime32Builder(mem, arrow.FixedWidthTypes.Time32ms.(*arrow.Time32Type)) 1047 defer bldr.Release() 1048 1049 vs := make([]arrow.Time32, len(a)) 1050 for i, v := range a { 1051 vs[i] = arrow.Time32(v) 1052 } 1053 bldr.AppendValues(vs, valids) 1054 return bldr.NewArray() 1055 1056 case []time64ns: 1057 bldr := array.NewTime64Builder(mem, arrow.FixedWidthTypes.Time64ns.(*arrow.Time64Type)) 1058 defer bldr.Release() 1059 1060 vs := make([]arrow.Time64, len(a)) 1061 for i, v := range a { 1062 vs[i] = arrow.Time64(v) 1063 } 1064 bldr.AppendValues(vs, valids) 1065 return bldr.NewArray() 1066 1067 case []time64us: 1068 bldr := array.NewTime64Builder(mem, arrow.FixedWidthTypes.Time64us.(*arrow.Time64Type)) 1069 defer bldr.Release() 1070 1071 vs := make([]arrow.Time64, len(a)) 1072 for i, v := range a { 1073 vs[i] = arrow.Time64(v) 1074 } 1075 bldr.AppendValues(vs, valids) 1076 return bldr.NewArray() 1077 1078 case []timestamp_s: 1079 bldr := array.NewTimestampBuilder(mem, arrow.FixedWidthTypes.Timestamp_s.(*arrow.TimestampType)) 1080 defer bldr.Release() 1081 1082 vs := make([]arrow.Timestamp, len(a)) 1083 for i, v := range a { 1084 vs[i] = arrow.Timestamp(v) 1085 } 1086 bldr.AppendValues(vs, valids) 1087 return bldr.NewArray() 1088 1089 case []timestamp_ms: 1090 bldr := array.NewTimestampBuilder(mem, arrow.FixedWidthTypes.Timestamp_ms.(*arrow.TimestampType)) 1091 defer bldr.Release() 1092 1093 vs := make([]arrow.Timestamp, len(a)) 1094 for i, v := range a { 1095 vs[i] = arrow.Timestamp(v) 1096 } 1097 bldr.AppendValues(vs, valids) 1098 return bldr.NewArray() 1099 1100 case []timestamp_us: 1101 bldr := array.NewTimestampBuilder(mem, arrow.FixedWidthTypes.Timestamp_us.(*arrow.TimestampType)) 1102 defer bldr.Release() 1103 1104 vs := make([]arrow.Timestamp, len(a)) 1105 for i, v := range a { 1106 vs[i] = arrow.Timestamp(v) 1107 } 1108 bldr.AppendValues(vs, valids) 1109 return bldr.NewArray() 1110 1111 case []timestamp_ns: 1112 bldr := array.NewTimestampBuilder(mem, arrow.FixedWidthTypes.Timestamp_ns.(*arrow.TimestampType)) 1113 defer bldr.Release() 1114 1115 vs := make([]arrow.Timestamp, len(a)) 1116 for i, v := range a { 1117 vs[i] = arrow.Timestamp(v) 1118 } 1119 bldr.AppendValues(vs, valids) 1120 return bldr.NewArray() 1121 1122 case []arrow.Date32: 1123 bldr := array.NewDate32Builder(mem) 1124 defer bldr.Release() 1125 1126 bldr.AppendValues(a, valids) 1127 return bldr.NewArray() 1128 1129 case []arrow.Date64: 1130 bldr := array.NewDate64Builder(mem) 1131 defer bldr.Release() 1132 1133 bldr.AppendValues(a, valids) 1134 return bldr.NewArray() 1135 1136 case []fsb3: 1137 bldr := array.NewFixedSizeBinaryBuilder(mem, &arrow.FixedSizeBinaryType{ByteWidth: 3}) 1138 defer bldr.Release() 1139 vs := make([][]byte, len(a)) 1140 for i, v := range a { 1141 vs[i] = []byte(v) 1142 } 1143 bldr.AppendValues(vs, valids) 1144 return bldr.NewArray() 1145 1146 case []arrow.MonthInterval: 1147 bldr := array.NewMonthIntervalBuilder(mem) 1148 defer bldr.Release() 1149 1150 bldr.AppendValues(a, valids) 1151 return bldr.NewArray() 1152 1153 case []arrow.DayTimeInterval: 1154 bldr := array.NewDayTimeIntervalBuilder(mem) 1155 defer bldr.Release() 1156 1157 bldr.AppendValues(a, valids) 1158 return bldr.NewArray() 1159 1160 case []arrow.MonthDayNanoInterval: 1161 bldr := array.NewMonthDayNanoIntervalBuilder(mem) 1162 defer bldr.Release() 1163 1164 bldr.AppendValues(a, valids) 1165 return bldr.NewArray() 1166 1167 case []duration_s: 1168 bldr := array.NewDurationBuilder(mem, &arrow.DurationType{Unit: arrow.Second}) 1169 defer bldr.Release() 1170 vs := make([]arrow.Duration, len(a)) 1171 for i, v := range a { 1172 vs[i] = arrow.Duration(v) 1173 } 1174 bldr.AppendValues(vs, valids) 1175 return bldr.NewArray() 1176 1177 case []duration_ms: 1178 bldr := array.NewDurationBuilder(mem, &arrow.DurationType{Unit: arrow.Millisecond}) 1179 defer bldr.Release() 1180 vs := make([]arrow.Duration, len(a)) 1181 for i, v := range a { 1182 vs[i] = arrow.Duration(v) 1183 } 1184 bldr.AppendValues(vs, valids) 1185 return bldr.NewArray() 1186 1187 case []duration_us: 1188 bldr := array.NewDurationBuilder(mem, &arrow.DurationType{Unit: arrow.Microsecond}) 1189 defer bldr.Release() 1190 vs := make([]arrow.Duration, len(a)) 1191 for i, v := range a { 1192 vs[i] = arrow.Duration(v) 1193 } 1194 bldr.AppendValues(vs, valids) 1195 return bldr.NewArray() 1196 1197 case []duration_ns: 1198 bldr := array.NewDurationBuilder(mem, &arrow.DurationType{Unit: arrow.Nanosecond}) 1199 defer bldr.Release() 1200 vs := make([]arrow.Duration, len(a)) 1201 for i, v := range a { 1202 vs[i] = arrow.Duration(v) 1203 } 1204 bldr.AppendValues(vs, valids) 1205 return bldr.NewArray() 1206 1207 default: 1208 panic(fmt.Errorf("arrdata: invalid data slice type %T", a)) 1209 } 1210} 1211 1212func listOf(mem memory.Allocator, values []array.Interface, valids []bool) *array.List { 1213 if mem == nil { 1214 mem = memory.NewGoAllocator() 1215 } 1216 1217 bldr := array.NewListBuilder(mem, values[0].DataType()) 1218 defer bldr.Release() 1219 1220 valid := func(i int) bool { 1221 return valids[i] 1222 } 1223 1224 if valids == nil { 1225 valid = func(i int) bool { return true } 1226 } 1227 1228 for i, value := range values { 1229 bldr.Append(valid(i)) 1230 buildArray(bldr.ValueBuilder(), value) 1231 } 1232 1233 return bldr.NewListArray() 1234} 1235 1236func fixedSizeListOf(mem memory.Allocator, n int32, values []array.Interface, valids []bool) *array.FixedSizeList { 1237 if mem == nil { 1238 mem = memory.NewGoAllocator() 1239 } 1240 1241 bldr := array.NewFixedSizeListBuilder(mem, n, values[0].DataType()) 1242 defer bldr.Release() 1243 1244 valid := func(i int) bool { 1245 return valids[i] 1246 } 1247 1248 if valids == nil { 1249 valid = func(i int) bool { return true } 1250 } 1251 1252 for i, value := range values { 1253 bldr.Append(valid(i)) 1254 buildArray(bldr.ValueBuilder(), value) 1255 } 1256 1257 return bldr.NewListArray() 1258} 1259 1260func structOf(mem memory.Allocator, dtype *arrow.StructType, fields [][]array.Interface, valids []bool) *array.Struct { 1261 if mem == nil { 1262 mem = memory.NewGoAllocator() 1263 } 1264 1265 bldr := array.NewStructBuilder(mem, dtype) 1266 defer bldr.Release() 1267 1268 if valids == nil { 1269 valids = make([]bool, fields[0][0].Len()) 1270 for i := range valids { 1271 valids[i] = true 1272 } 1273 } 1274 1275 for i := range fields { 1276 bldr.AppendValues(valids) 1277 for j := range dtype.Fields() { 1278 fbldr := bldr.FieldBuilder(j) 1279 buildArray(fbldr, fields[i][j]) 1280 } 1281 } 1282 1283 return bldr.NewStructArray() 1284} 1285 1286func mapOf(mem memory.Allocator, sortedKeys bool, values []array.Interface, valids []bool) *array.Map { 1287 if mem == nil { 1288 mem = memory.NewGoAllocator() 1289 } 1290 1291 pairType := values[0].DataType().(*arrow.StructType) 1292 bldr := array.NewMapBuilder(mem, pairType.Field(0).Type, pairType.Field(1).Type, sortedKeys) 1293 defer bldr.Release() 1294 1295 valid := func(i int) bool { 1296 return valids[i] 1297 } 1298 1299 if valids == nil { 1300 valid = func(i int) bool { return true } 1301 } 1302 1303 vb := bldr.ValueBuilder() 1304 for i, value := range values { 1305 bldr.Append(valid(i)) 1306 buildArray(vb.FieldBuilder(0), value.(*array.Struct).Field(0)) 1307 buildArray(vb.FieldBuilder(1), value.(*array.Struct).Field(1)) 1308 } 1309 1310 return bldr.NewMapArray() 1311} 1312 1313func buildArray(bldr array.Builder, data array.Interface) { 1314 defer data.Release() 1315 1316 switch bldr := bldr.(type) { 1317 case *array.BooleanBuilder: 1318 data := data.(*array.Boolean) 1319 for i := 0; i < data.Len(); i++ { 1320 switch { 1321 case data.IsValid(i): 1322 bldr.Append(data.Value(i)) 1323 default: 1324 bldr.AppendNull() 1325 } 1326 } 1327 1328 case *array.Int8Builder: 1329 data := data.(*array.Int8) 1330 for i := 0; i < data.Len(); i++ { 1331 switch { 1332 case data.IsValid(i): 1333 bldr.Append(data.Value(i)) 1334 default: 1335 bldr.AppendNull() 1336 } 1337 } 1338 1339 case *array.Int16Builder: 1340 data := data.(*array.Int16) 1341 for i := 0; i < data.Len(); i++ { 1342 switch { 1343 case data.IsValid(i): 1344 bldr.Append(data.Value(i)) 1345 default: 1346 bldr.AppendNull() 1347 } 1348 } 1349 1350 case *array.Int32Builder: 1351 data := data.(*array.Int32) 1352 for i := 0; i < data.Len(); i++ { 1353 switch { 1354 case data.IsValid(i): 1355 bldr.Append(data.Value(i)) 1356 default: 1357 bldr.AppendNull() 1358 } 1359 } 1360 1361 case *array.Int64Builder: 1362 data := data.(*array.Int64) 1363 for i := 0; i < data.Len(); i++ { 1364 switch { 1365 case data.IsValid(i): 1366 bldr.Append(data.Value(i)) 1367 default: 1368 bldr.AppendNull() 1369 } 1370 } 1371 1372 case *array.Uint8Builder: 1373 data := data.(*array.Uint8) 1374 for i := 0; i < data.Len(); i++ { 1375 switch { 1376 case data.IsValid(i): 1377 bldr.Append(data.Value(i)) 1378 default: 1379 bldr.AppendNull() 1380 } 1381 } 1382 1383 case *array.Uint16Builder: 1384 data := data.(*array.Uint16) 1385 for i := 0; i < data.Len(); i++ { 1386 switch { 1387 case data.IsValid(i): 1388 bldr.Append(data.Value(i)) 1389 default: 1390 bldr.AppendNull() 1391 } 1392 } 1393 1394 case *array.Uint32Builder: 1395 data := data.(*array.Uint32) 1396 for i := 0; i < data.Len(); i++ { 1397 switch { 1398 case data.IsValid(i): 1399 bldr.Append(data.Value(i)) 1400 default: 1401 bldr.AppendNull() 1402 } 1403 } 1404 1405 case *array.Uint64Builder: 1406 data := data.(*array.Uint64) 1407 for i := 0; i < data.Len(); i++ { 1408 switch { 1409 case data.IsValid(i): 1410 bldr.Append(data.Value(i)) 1411 default: 1412 bldr.AppendNull() 1413 } 1414 } 1415 1416 case *array.Float32Builder: 1417 data := data.(*array.Float32) 1418 for i := 0; i < data.Len(); i++ { 1419 switch { 1420 case data.IsValid(i): 1421 bldr.Append(data.Value(i)) 1422 default: 1423 bldr.AppendNull() 1424 } 1425 } 1426 1427 case *array.Float64Builder: 1428 data := data.(*array.Float64) 1429 for i := 0; i < data.Len(); i++ { 1430 switch { 1431 case data.IsValid(i): 1432 bldr.Append(data.Value(i)) 1433 default: 1434 bldr.AppendNull() 1435 } 1436 } 1437 1438 case *array.StringBuilder: 1439 data := data.(*array.String) 1440 for i := 0; i < data.Len(); i++ { 1441 switch { 1442 case data.IsValid(i): 1443 bldr.Append(data.Value(i)) 1444 default: 1445 bldr.AppendNull() 1446 } 1447 } 1448 } 1449} 1450