1# Licensed to the Apache Software Foundation (ASF) under one 2# or more contributor license agreements. See the NOTICE file 3# distributed with this work for additional information 4# regarding copyright ownership. The ASF licenses this file 5# to you under the Apache License, Version 2.0 (the 6# "License"); you may not use this file except in compliance 7# with the License. You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, 12# software distributed under the License is distributed on an 13# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14# KIND, either express or implied. See the License for the 15# specific language governing permissions and limitations 16# under the License. 17 18module RawRecordsListArrayTests 19 def build_schema(type) 20 field_description = { 21 name: :element, 22 } 23 if type.is_a?(Hash) 24 field_description = field_description.merge(type) 25 else 26 field_description[:type] = type 27 end 28 { 29 column: { 30 type: :list, 31 field: field_description, 32 }, 33 } 34 end 35 36 def test_null 37 records = [ 38 [[nil, nil, nil]], 39 [nil], 40 ] 41 target = build(:null, records) 42 assert_equal(records, target.raw_records) 43 end 44 45 def test_boolean 46 records = [ 47 [[true, nil, false]], 48 [nil], 49 ] 50 target = build(:boolean, records) 51 assert_equal(records, target.raw_records) 52 end 53 54 def test_int8 55 records = [ 56 [[-(2 ** 7), nil, (2 ** 7) - 1]], 57 [nil], 58 ] 59 target = build(:int8, records) 60 assert_equal(records, target.raw_records) 61 end 62 63 def test_uint8 64 records = [ 65 [[0, nil, (2 ** 8) - 1]], 66 [nil], 67 ] 68 target = build(:uint8, records) 69 assert_equal(records, target.raw_records) 70 end 71 72 def test_int16 73 records = [ 74 [[-(2 ** 15), nil, (2 ** 15) - 1]], 75 [nil], 76 ] 77 target = build(:int16, records) 78 assert_equal(records, target.raw_records) 79 end 80 81 def test_uint16 82 records = [ 83 [[0, nil, (2 ** 16) - 1]], 84 [nil], 85 ] 86 target = build(:uint16, records) 87 assert_equal(records, target.raw_records) 88 end 89 90 def test_int32 91 records = [ 92 [[-(2 ** 31), nil, (2 ** 31) - 1]], 93 [nil], 94 ] 95 target = build(:int32, records) 96 assert_equal(records, target.raw_records) 97 end 98 99 def test_uint32 100 records = [ 101 [[0, nil, (2 ** 32) - 1]], 102 [nil], 103 ] 104 target = build(:uint32, records) 105 assert_equal(records, target.raw_records) 106 end 107 108 def test_int64 109 records = [ 110 [[-(2 ** 63), nil, (2 ** 63) - 1]], 111 [nil], 112 ] 113 target = build(:int64, records) 114 assert_equal(records, target.raw_records) 115 end 116 117 def test_uint64 118 records = [ 119 [[0, nil, (2 ** 64) - 1]], 120 [nil], 121 ] 122 target = build(:uint64, records) 123 assert_equal(records, target.raw_records) 124 end 125 126 def test_float 127 records = [ 128 [[-1.0, nil, 1.0]], 129 [nil], 130 ] 131 target = build(:float, records) 132 assert_equal(records, target.raw_records) 133 end 134 135 def test_double 136 records = [ 137 [[-1.0, nil, 1.0]], 138 [nil], 139 ] 140 target = build(:double, records) 141 assert_equal(records, target.raw_records) 142 end 143 144 def test_binary 145 records = [ 146 [["\x00".b, nil, "\xff".b]], 147 [nil], 148 ] 149 target = build(:binary, records) 150 assert_equal(records, target.raw_records) 151 end 152 153 def test_string 154 records = [ 155 [ 156 [ 157 "Ruby", 158 nil, 159 "\u3042", # U+3042 HIRAGANA LETTER A 160 ], 161 ], 162 [nil], 163 ] 164 target = build(:string, records) 165 assert_equal(records, target.raw_records) 166 end 167 168 def test_date32 169 records = [ 170 [ 171 [ 172 Date.new(1960, 1, 1), 173 nil, 174 Date.new(2017, 8, 23), 175 ], 176 ], 177 [nil], 178 ] 179 target = build(:date32, records) 180 assert_equal(records, target.raw_records) 181 end 182 183 def test_date64 184 records = [ 185 [ 186 [ 187 DateTime.new(1960, 1, 1, 2, 9, 30), 188 nil, 189 DateTime.new(2017, 8, 23, 14, 57, 2), 190 ], 191 ], 192 [nil], 193 ] 194 target = build(:date64, records) 195 assert_equal(records, target.raw_records) 196 end 197 198 def test_timestamp_second 199 records = [ 200 [ 201 [ 202 Time.parse("1960-01-01T02:09:30Z"), 203 nil, 204 Time.parse("2017-08-23T14:57:02Z"), 205 ], 206 ], 207 [nil], 208 ] 209 target = build({ 210 type: :timestamp, 211 unit: :second, 212 }, 213 records) 214 assert_equal(records, target.raw_records) 215 end 216 217 def test_timestamp_milli 218 records = [ 219 [ 220 [ 221 Time.parse("1960-01-01T02:09:30.123Z"), 222 nil, 223 Time.parse("2017-08-23T14:57:02.987Z"), 224 ], 225 ], 226 [nil], 227 ] 228 target = build({ 229 type: :timestamp, 230 unit: :milli, 231 }, 232 records) 233 assert_equal(records, target.raw_records) 234 end 235 236 def test_timestamp_micro 237 records = [ 238 [ 239 [ 240 Time.parse("1960-01-01T02:09:30.123456Z"), 241 nil, 242 Time.parse("2017-08-23T14:57:02.987654Z"), 243 ], 244 ], 245 [nil], 246 ] 247 target = build({ 248 type: :timestamp, 249 unit: :micro, 250 }, 251 records) 252 assert_equal(records, target.raw_records) 253 end 254 255 def test_timestamp_nano 256 records = [ 257 [ 258 [ 259 Time.parse("1960-01-01T02:09:30.123456789Z"), 260 nil, 261 Time.parse("2017-08-23T14:57:02.987654321Z"), 262 ], 263 ], 264 [nil], 265 ] 266 target = build({ 267 type: :timestamp, 268 unit: :nano, 269 }, 270 records) 271 assert_equal(records, target.raw_records) 272 end 273 274 def test_time32_second 275 unit = Arrow::TimeUnit::SECOND 276 records = [ 277 [ 278 [ 279 # 00:10:00 280 Arrow::Time.new(unit, 60 * 10), 281 nil, 282 # 02:00:09 283 Arrow::Time.new(unit, 60 * 60 * 2 + 9), 284 ], 285 ], 286 [nil], 287 ] 288 target = build({ 289 type: :time32, 290 unit: :second, 291 }, 292 records) 293 assert_equal(records, target.raw_records) 294 end 295 296 def test_time32_milli 297 unit = Arrow::TimeUnit::MILLI 298 records = [ 299 [ 300 [ 301 # 00:10:00.123 302 Arrow::Time.new(unit, (60 * 10) * 1000 + 123), 303 nil, 304 # 02:00:09.987 305 Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1000 + 987), 306 ], 307 ], 308 [nil], 309 ] 310 target = build({ 311 type: :time32, 312 unit: :milli, 313 }, 314 records) 315 assert_equal(records, target.raw_records) 316 end 317 318 def test_time64_micro 319 unit = Arrow::TimeUnit::MICRO 320 records = [ 321 [ 322 [ 323 # 00:10:00.123456 324 Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456), 325 nil, 326 # 02:00:09.987654 327 Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000 + 987_654), 328 ], 329 ], 330 [nil], 331 ] 332 target = build({ 333 type: :time64, 334 unit: :micro, 335 }, 336 records) 337 assert_equal(records, target.raw_records) 338 end 339 340 def test_time64_nano 341 unit = Arrow::TimeUnit::NANO 342 records = [ 343 [ 344 [ 345 # 00:10:00.123456789 346 Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789), 347 nil, 348 # 02:00:09.987654321 349 Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000_000 + 987_654_321), 350 ], 351 ], 352 [nil], 353 ] 354 target = build({ 355 type: :time64, 356 unit: :nano, 357 }, 358 records) 359 assert_equal(records, target.raw_records) 360 end 361 362 def test_decimal128 363 records = [ 364 [ 365 [ 366 BigDecimal("92.92"), 367 nil, 368 BigDecimal("29.29"), 369 ], 370 ], 371 [nil], 372 ] 373 target = build({ 374 type: :decimal128, 375 precision: 8, 376 scale: 2, 377 }, 378 records) 379 assert_equal(records, target.raw_records) 380 end 381 382 def test_decimal256 383 records = [ 384 [ 385 [ 386 BigDecimal("92.92"), 387 nil, 388 BigDecimal("29.29"), 389 ], 390 ], 391 [nil], 392 ] 393 target = build({ 394 type: :decimal256, 395 precision: 38, 396 scale: 2, 397 }, 398 records) 399 assert_equal(records, target.raw_records) 400 end 401 402 def test_list 403 records = [ 404 [ 405 [ 406 [ 407 true, 408 nil, 409 ], 410 nil, 411 [ 412 nil, 413 false, 414 ], 415 ], 416 ], 417 [nil], 418 ] 419 target = build({ 420 type: :list, 421 field: { 422 name: :sub_element, 423 type: :boolean, 424 }, 425 }, 426 records) 427 assert_equal(records, target.raw_records) 428 end 429 430 def test_struct 431 records = [ 432 [ 433 [ 434 {"field" => true}, 435 nil, 436 {"field" => nil}, 437 ], 438 ], 439 [nil], 440 ] 441 target = build({ 442 type: :struct, 443 fields: [ 444 { 445 name: :field, 446 type: :boolean, 447 }, 448 ], 449 }, 450 records) 451 assert_equal(records, target.raw_records) 452 end 453 454 def test_sparse 455 omit("Need to add support for SparseUnionArrayBuilder") 456 records = [ 457 [ 458 [ 459 {"field1" => true}, 460 nil, 461 {"field2" => nil}, 462 ], 463 ], 464 [nil], 465 ] 466 target = build({ 467 type: :sparse_union, 468 fields: [ 469 { 470 name: :field1, 471 type: :boolean, 472 }, 473 { 474 name: :field2, 475 type: :uint8, 476 }, 477 ], 478 type_codes: [0, 1], 479 }, 480 records) 481 assert_equal(records, target.raw_records) 482 end 483 484 def test_dense 485 omit("Need to add support for DenseUnionArrayBuilder") 486 records = [ 487 [ 488 [ 489 {"field1" => true}, 490 nil, 491 {"field2" => nil}, 492 ], 493 ], 494 [nil], 495 ] 496 target = build({ 497 type: :dense_union, 498 fields: [ 499 { 500 name: :field1, 501 type: :boolean, 502 }, 503 { 504 name: :field2, 505 type: :uint8, 506 }, 507 ], 508 type_codes: [0, 1], 509 }, 510 records) 511 assert_equal(records, target.raw_records) 512 end 513 514 def test_dictionary 515 omit("Need to add support for DictionaryArrayBuilder") 516 records = [ 517 [ 518 [ 519 "Ruby", 520 nil, 521 "GLib", 522 ], 523 ], 524 [nil], 525 ] 526 dictionary = Arrow::StringArray.new(["GLib", "Ruby"]) 527 target = build({ 528 type: :dictionary, 529 index_data_type: :int8, 530 dictionary: dictionary, 531 ordered: true, 532 }, 533 records) 534 assert_equal(records, target.raw_records) 535 end 536end 537 538class RawRecordsRecordBatchListArrayTest < Test::Unit::TestCase 539 include RawRecordsListArrayTests 540 541 def build(type, records) 542 Arrow::RecordBatch.new(build_schema(type), records) 543 end 544end 545 546class RawRecordsTableListArrayTest < Test::Unit::TestCase 547 include RawRecordsListArrayTests 548 549 def build(type, records) 550 Arrow::Table.new(build_schema(type), records) 551 end 552end 553