1# Licensed to the Apache Software Foundation (ASF) under one 2# or more contributor license agreements. See the NOTICE file 3# distributed with this work for additional information 4# regarding copyright ownership. The ASF licenses this file 5# to you under the Apache License, Version 2.0 (the 6# "License"); you may not use this file except in compliance 7# with the License. You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, 12# software distributed under the License is distributed on an 13# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14# KIND, either express or implied. See the License for the 15# specific language governing permissions and limitations 16# under the License. 17 18require "cgi/util" 19require "digest/sha2" 20require "io/console" 21require "json" 22require "net/http" 23require "pathname" 24require "tempfile" 25require "thread" 26require "time" 27 28begin 29 require "apt-dists-merge" 30rescue LoadError 31 warn("apt-dists-merge is needed for apt:* tasks") 32end 33 34class BinaryTask 35 include Rake::DSL 36 37 class ThreadPool 38 def initialize(use_case, &worker) 39 @n_workers = choose_n_workers(use_case) 40 @worker = worker 41 @jobs = Thread::Queue.new 42 @workers = @n_workers.times.collect do 43 Thread.new do 44 loop do 45 job = @jobs.pop 46 break if job.nil? 47 @worker.call(job) 48 end 49 end 50 end 51 end 52 53 def <<(job) 54 @jobs << job 55 end 56 57 def join 58 @n_workers.times do 59 @jobs << nil 60 end 61 @workers.each(&:join) 62 end 63 64 private 65 def choose_n_workers(use_case) 66 case use_case 67 when :artifactory 68 # Too many workers cause Artifactory error. 69 6 70 when :gpg 71 # Too many workers cause gpg-agent error. 72 2 73 else 74 raise "Unknown use case: #{use_case}" 75 end 76 end 77 end 78 79 class ProgressReporter 80 def initialize(label, count_max=0) 81 @label = label 82 @count_max = count_max 83 84 @mutex = Thread::Mutex.new 85 86 @time_start = Time.now 87 @time_previous = Time.now 88 @count_current = 0 89 @count_previous = 0 90 end 91 92 def advance 93 @mutex.synchronize do 94 @count_current += 1 95 96 return if @count_max.zero? 97 98 time_current = Time.now 99 if time_current - @time_previous <= 1 100 return 101 end 102 103 show_progress(time_current) 104 end 105 end 106 107 def increment_max 108 @mutex.synchronize do 109 @count_max += 1 110 show_progress(Time.now) if @count_max == 1 111 end 112 end 113 114 def finish 115 @mutex.synchronize do 116 return if @count_max.zero? 117 show_progress(Time.now) 118 $stderr.puts 119 end 120 end 121 122 private 123 def show_progress(time_current) 124 n_finishes = @count_current - @count_previous 125 throughput = n_finishes.to_f / (time_current - @time_previous) 126 @time_previous = time_current 127 @count_previous = @count_current 128 129 message = build_message(time_current, throughput) 130 $stderr.print("\r#{message}") if message 131 end 132 133 def build_message(time_current, throughput) 134 percent = (@count_current / @count_max.to_f) * 100 135 formatted_count = "[%s/%s]" % [ 136 format_count(@count_current), 137 format_count(@count_max), 138 ] 139 elapsed_second = time_current - @time_start 140 if throughput.zero? 141 rest_second = 0 142 else 143 rest_second = (@count_max - @count_current) / throughput 144 end 145 separator = " - " 146 progress = "%5.1f%% %s %s %s %s" % [ 147 percent, 148 formatted_count, 149 format_time_interval(elapsed_second), 150 format_time_interval(rest_second), 151 format_throughput(throughput), 152 ] 153 label = @label 154 155 width = guess_terminal_width 156 return "#{label}#{separator}#{progress}" if width.nil? 157 158 return nil if progress.size > width 159 160 label_width = width - progress.size - separator.size 161 if label.size > label_width 162 ellipsis = "..." 163 shorten_label_width = label_width - ellipsis.size 164 if shorten_label_width < 1 165 return progress 166 else 167 label = label[0, shorten_label_width] + ellipsis 168 end 169 end 170 "#{label}#{separator}#{progress}" 171 end 172 173 def format_count(count) 174 "%d" % count 175 end 176 177 def format_time_interval(interval) 178 if interval < 60 179 "00:00:%02d" % interval 180 elsif interval < (60 * 60) 181 minute, second = interval.divmod(60) 182 "00:%02d:%02d" % [minute, second] 183 elsif interval < (60 * 60 * 24) 184 minute, second = interval.divmod(60) 185 hour, minute = minute.divmod(60) 186 "%02d:%02d:%02d" % [hour, minute, second] 187 else 188 minute, second = interval.divmod(60) 189 hour, minute = minute.divmod(60) 190 day, hour = hour.divmod(24) 191 "%dd %02d:%02d:%02d" % [day, hour, minute, second] 192 end 193 end 194 195 def format_throughput(throughput) 196 "%2d/s" % throughput 197 end 198 199 def guess_terminal_width 200 guess_terminal_width_from_io || 201 guess_terminal_width_from_command || 202 guess_terminal_width_from_env || 203 80 204 end 205 206 def guess_terminal_width_from_io 207 if IO.respond_to?(:console) and IO.console 208 IO.console.winsize[1] 209 elsif $stderr.respond_to?(:winsize) 210 begin 211 $stderr.winsize[1] 212 rescue SystemCallError 213 nil 214 end 215 else 216 nil 217 end 218 end 219 220 def guess_terminal_width_from_command 221 IO.pipe do |input, output| 222 begin 223 pid = spawn("tput", "cols", {:out => output, :err => output}) 224 rescue SystemCallError 225 return nil 226 end 227 228 output.close 229 _, status = Process.waitpid2(pid) 230 return nil unless status.success? 231 232 result = input.read.chomp 233 begin 234 Integer(result, 10) 235 rescue ArgumentError 236 nil 237 end 238 end 239 end 240 241 def guess_terminal_width_from_env 242 env = ENV["COLUMNS"] || ENV["TERM_WIDTH"] 243 return nil if env.nil? 244 245 begin 246 Integer(env, 10) 247 rescue ArgumentError 248 nil 249 end 250 end 251 end 252 253 class ArtifactoryClient 254 class Error < StandardError 255 attr_reader :request 256 attr_reader :response 257 def initialize(request, response, message) 258 @request = request 259 @response = response 260 super(message) 261 end 262 end 263 264 def initialize(prefix, api_key) 265 @prefix = prefix 266 @api_key = api_key 267 @http = nil 268 restart 269 end 270 271 def restart 272 close 273 @http = start_http(build_url("")) 274 end 275 276 private def start_http(url, &block) 277 http = Net::HTTP.new(url.host, url.port) 278 http.set_debug_output($stderr) if ENV["DEBUG"] 279 http.use_ssl = true 280 if block_given? 281 http.start(&block) 282 else 283 http 284 end 285 end 286 287 def close 288 return if @http.nil? 289 @http.finish if @http.started? 290 @http = nil 291 end 292 293 def request(method, headers, url, body: nil, &block) 294 request = build_request(method, url, headers, body: body) 295 if ENV["DRY_RUN"] 296 case request 297 when Net::HTTP::Get, Net::HTTP::Head 298 else 299 p [method, url] 300 return 301 end 302 end 303 request_internal(@http, request, &block) 304 end 305 306 private def request_internal(http, request, &block) 307 http.request(request) do |response| 308 case response 309 when Net::HTTPSuccess, 310 Net::HTTPNotModified 311 if block_given? 312 return yield(response) 313 else 314 response.read_body 315 return response 316 end 317 when Net::HTTPRedirection 318 redirected_url = URI(response["Location"]) 319 redirected_request = Net::HTTP::Get.new(redirected_url, {}) 320 start_http(redirected_url) do |redirected_http| 321 request_internal(redirected_http, redirected_request, &block) 322 end 323 else 324 message = "failed to request: " 325 message << "#{request.uri}: #{request.method}: " 326 message << "#{response.message} #{response.code}" 327 if response.body 328 message << "\n" 329 message << response.body 330 end 331 raise Error.new(request, response, message) 332 end 333 end 334 end 335 336 def files 337 _files = [] 338 directories = [""] 339 until directories.empty? 340 directory = directories.shift 341 list(directory).each do |path| 342 resolved_path = "#{directory}#{path}" 343 case path 344 when "../" 345 when /\/\z/ 346 directories << resolved_path 347 else 348 _files << resolved_path 349 end 350 end 351 end 352 _files 353 end 354 355 def list(path) 356 url = build_url(path) 357 with_retry(3, url) do 358 begin 359 request(:get, {}, url) do |response| 360 response.body.scan(/<a href="(.+?)"/).flatten 361 end 362 rescue Error => error 363 case error.response 364 when Net::HTTPNotFound 365 return [] 366 else 367 raise 368 end 369 end 370 end 371 end 372 373 def head(path) 374 url = build_url(path) 375 with_retry(3, url) do 376 request(:head, {}, url) 377 end 378 end 379 380 def exist?(path) 381 begin 382 head(path) 383 true 384 rescue Error => error 385 case error.response 386 when Net::HTTPNotFound 387 false 388 else 389 raise 390 end 391 end 392 end 393 394 def upload(path, destination_path) 395 destination_url = build_url(destination_path) 396 with_retry(3, destination_url) do 397 sha1 = Digest::SHA1.file(path).hexdigest 398 sha256 = Digest::SHA256.file(path).hexdigest 399 headers = { 400 "X-Artifactory-Last-Modified" => File.mtime(path).rfc2822, 401 "X-Checksum-Deploy" => "false", 402 "X-Checksum-Sha1" => sha1, 403 "X-Checksum-Sha256" => sha256, 404 "Content-Length" => File.size(path).to_s, 405 "Content-Type" => "application/octet-stream", 406 } 407 File.open(path, "rb") do |input| 408 request(:put, headers, destination_url, body: input) 409 end 410 end 411 end 412 413 def download(path, output_path) 414 url = build_url(path) 415 with_retry(5, url) do 416 begin 417 begin 418 headers = {} 419 if File.exist?(output_path) 420 headers["If-Modified-Since"] = File.mtime(output_path).rfc2822 421 end 422 request(:get, headers, url) do |response| 423 case response 424 when Net::HTTPNotModified 425 else 426 File.open(output_path, "wb") do |output| 427 response.read_body do |chunk| 428 output.write(chunk) 429 end 430 end 431 last_modified = response["Last-Modified"] 432 if last_modified 433 FileUtils.touch(output_path, 434 mtime: Time.rfc2822(last_modified)) 435 end 436 end 437 end 438 rescue Error => error 439 case error.response 440 when Net::HTTPNotFound 441 $stderr.puts(error.message) 442 return 443 else 444 raise 445 end 446 end 447 end 448 rescue 449 FileUtils.rm_f(output_path) 450 raise 451 end 452 end 453 454 def delete(path) 455 url = build_url(path) 456 with_retry(3, url) do 457 request(:delete, {}, url) 458 end 459 end 460 461 def copy(source, destination) 462 uri = build_api_url("copy/arrow/#{source}", 463 "to" => "/arrow/#{destination}") 464 with_read_timeout(300) do 465 request(:post, {}, uri) 466 end 467 end 468 469 private 470 def build_url(path) 471 uri_string = "https://apache.jfrog.io/artifactory/arrow" 472 uri_string << "/#{@prefix}" unless @prefix.nil? 473 uri_string << "/#{path}" 474 URI(uri_string) 475 end 476 477 def build_api_url(path, parameters) 478 uri_string = "https://apache.jfrog.io/artifactory/api/#{path}" 479 unless parameters.empty? 480 uri_string << "?" 481 escaped_parameters = parameters.collect do |key, value| 482 "#{CGI.escape(key)}=#{CGI.escape(value)}" 483 end 484 uri_string << escaped_parameters.join("&") 485 end 486 URI(uri_string) 487 end 488 489 def build_request(method, url, headers, body: nil) 490 need_auth = false 491 case method 492 when :head 493 request = Net::HTTP::Head.new(url, headers) 494 when :get 495 request = Net::HTTP::Get.new(url, headers) 496 when :post 497 need_auth = true 498 request = Net::HTTP::Post.new(url, headers) 499 when :put 500 need_auth = true 501 request = Net::HTTP::Put.new(url, headers) 502 when :delete 503 need_auth = true 504 request = Net::HTTP::Delete.new(url, headers) 505 else 506 raise "unsupported HTTP method: #{method.inspect}" 507 end 508 request["Connection"] = "Keep-Alive" 509 request["X-JFrog-Art-Api"] = @api_key if need_auth 510 if body 511 if body.is_a?(String) 512 request.body = body 513 else 514 request.body_stream = body 515 end 516 end 517 request 518 end 519 520 def with_retry(max_n_retries, target) 521 n_retries = 0 522 begin 523 yield 524 rescue Net::OpenTimeout, 525 OpenSSL::OpenSSLError, 526 SocketError, 527 SystemCallError, 528 Timeout::Error => error 529 n_retries += 1 530 if n_retries <= max_n_retries 531 $stderr.puts 532 $stderr.puts("Retry #{n_retries}: #{target}: " + 533 "#{error.class}: #{error.message}") 534 restart 535 retry 536 else 537 raise 538 end 539 end 540 end 541 542 def with_read_timeout(timeout) 543 current_timeout = @http.read_timeout 544 begin 545 @http.read_timeout = timeout 546 yield 547 ensure 548 @http.read_timeout = current_timeout 549 end 550 end 551 end 552 553 class ArtifactoryClientPool 554 class << self 555 def open(prefix, api_key) 556 pool = new(prefix, api_key) 557 begin 558 yield(pool) 559 ensure 560 pool.close 561 end 562 end 563 end 564 565 def initialize(prefix, api_key) 566 @prefix = prefix 567 @api_key = api_key 568 @mutex = Thread::Mutex.new 569 @clients = [] 570 end 571 572 def pull 573 client = @mutex.synchronize do 574 if @clients.empty? 575 ArtifactoryClient.new(@prefix, @api_key) 576 else 577 @clients.pop 578 end 579 end 580 begin 581 yield(client) 582 ensure 583 release(client) 584 end 585 end 586 587 def release(client) 588 @mutex.synchronize do 589 @clients << client 590 end 591 end 592 593 def close 594 @clients.each(&:close) 595 end 596 end 597 598 module ArtifactoryPath 599 private 600 def base_path 601 path = @distribution 602 path += "-staging" if @staging 603 path += "-rc" if @rc 604 path 605 end 606 end 607 608 class ArtifactoryDownloader 609 include ArtifactoryPath 610 611 def initialize(api_key:, 612 destination:, 613 distribution:, 614 list: nil, 615 pattern: nil, 616 prefix: nil, 617 rc: nil, 618 staging: false) 619 @api_key = api_key 620 @destination = destination 621 @distribution = distribution 622 @list = list 623 @pattern = pattern 624 @prefix = prefix 625 @rc = rc 626 @staging = staging 627 end 628 629 def download 630 progress_label = "Downloading: #{base_path}" 631 progress_reporter = ProgressReporter.new(progress_label) 632 prefix = [base_path, @prefix].compact.join("/") 633 ArtifactoryClientPool.open(prefix, @api_key) do |client_pool| 634 thread_pool = ThreadPool.new(:artifactory) do |path, output_path| 635 client_pool.pull do |client| 636 client.download(path, output_path) 637 end 638 progress_reporter.advance 639 end 640 files = client_pool.pull do |client| 641 if @list 642 list_output_path = "#{@destination}/#{@list}" 643 client.download(@list, list_output_path) 644 File.readlines(list_output_path, chomp: true) 645 else 646 client.files 647 end 648 end 649 files.each do |path| 650 output_path = "#{@destination}/#{path}" 651 if @pattern 652 next unless @pattern.match?(path) 653 end 654 yield(output_path) 655 output_dir = File.dirname(output_path) 656 FileUtils.mkdir_p(output_dir) 657 progress_reporter.increment_max 658 thread_pool << [path, output_path] 659 end 660 thread_pool.join 661 end 662 progress_reporter.finish 663 end 664 end 665 666 class ArtifactoryUploader 667 include ArtifactoryPath 668 669 def initialize(api_key:, 670 destination_prefix: nil, 671 distribution:, 672 rc: nil, 673 source:, 674 staging: false, 675 sync: false, 676 sync_pattern: nil) 677 @api_key = api_key 678 @destination_prefix = destination_prefix 679 @distribution = distribution 680 @rc = rc 681 @source = source 682 @staging = staging 683 @sync = sync 684 @sync_pattern = sync_pattern 685 end 686 687 def upload 688 progress_label = "Uploading: #{base_path}" 689 progress_reporter = ProgressReporter.new(progress_label) 690 prefix = base_path 691 prefix += "/#{@destination_prefix}" if @destination_prefix 692 ArtifactoryClientPool.open(prefix, @api_key) do |client_pool| 693 if @sync 694 existing_files = client_pool.pull do |client| 695 client.files 696 end 697 else 698 existing_files = [] 699 end 700 701 thread_pool = ThreadPool.new(:artifactory) do |path, relative_path| 702 client_pool.pull do |client| 703 client.upload(path, relative_path) 704 end 705 progress_reporter.advance 706 end 707 708 source = Pathname(@source) 709 source.glob("**/*") do |path| 710 next if path.directory? 711 destination_path = path.relative_path_from(source) 712 progress_reporter.increment_max 713 existing_files.delete(destination_path.to_s) 714 thread_pool << [path, destination_path] 715 end 716 thread_pool.join 717 718 if @sync 719 thread_pool = ThreadPool.new(:artifactory) do |path| 720 client_pool.pull do |client| 721 client.delete(path) 722 end 723 progress_reporter.advance 724 end 725 existing_files.each do |path| 726 if @sync_pattern 727 next unless @sync_pattern.match?(path) 728 end 729 progress_reporter.increment_max 730 thread_pool << path 731 end 732 thread_pool.join 733 end 734 end 735 progress_reporter.finish 736 end 737 end 738 739 def define 740 define_apt_tasks 741 define_yum_tasks 742 define_python_tasks 743 define_nuget_tasks 744 define_summary_tasks 745 end 746 747 private 748 def env_value(name) 749 value = ENV[name] 750 value = yield(name) if value.nil? and block_given? 751 raise "Specify #{name} environment variable" if value.nil? 752 value 753 end 754 755 def verbose? 756 ENV["VERBOSE"] == "yes" 757 end 758 759 def default_output 760 if verbose? 761 $stdout 762 else 763 IO::NULL 764 end 765 end 766 767 def gpg_key_id 768 env_value("GPG_KEY_ID") 769 end 770 771 def shorten_gpg_key_id(id) 772 id[-8..-1] 773 end 774 775 def rpm_gpg_key_package_name(id) 776 "gpg-pubkey-#{shorten_gpg_key_id(id).downcase}" 777 end 778 779 def artifactory_api_key 780 env_value("ARTIFACTORY_API_KEY") 781 end 782 783 def artifacts_dir 784 env_value("ARTIFACTS_DIR") 785 end 786 787 def version 788 env_value("VERSION") 789 end 790 791 def rc 792 env_value("RC") 793 end 794 795 def staging? 796 ENV["STAGING"] == "yes" 797 end 798 799 def full_version 800 "#{version}-rc#{rc}" 801 end 802 803 def valid_sign?(path, sign_path) 804 IO.pipe do |input, output| 805 begin 806 sh({"LANG" => "C"}, 807 "gpg", 808 "--verify", 809 sign_path, 810 path, 811 out: default_output, 812 err: output, 813 verbose: false) 814 rescue 815 return false 816 end 817 output.close 818 /Good signature/ === input.read 819 end 820 end 821 822 def sign(source_path, destination_path) 823 if File.exist?(destination_path) 824 return if valid_sign?(source_path, destination_path) 825 rm(destination_path, verbose: false) 826 end 827 sh("gpg", 828 "--detach-sig", 829 "--local-user", gpg_key_id, 830 "--output", destination_path, 831 source_path, 832 out: default_output, 833 verbose: verbose?) 834 end 835 836 def sha512(source_path, destination_path) 837 if File.exist?(destination_path) 838 sha512 = File.read(destination_path).split[0] 839 return if Digest::SHA512.file(source_path).hexdigest == sha512 840 end 841 absolute_destination_path = File.expand_path(destination_path) 842 Dir.chdir(File.dirname(source_path)) do 843 sh("shasum", 844 "--algorithm", "512", 845 File.basename(source_path), 846 out: absolute_destination_path, 847 verbose: verbose?) 848 end 849 end 850 851 def sign_dir(label, dir) 852 progress_label = "Signing: #{label}" 853 progress_reporter = ProgressReporter.new(progress_label) 854 855 target_paths = [] 856 Pathname(dir).glob("**/*") do |path| 857 next if path.directory? 858 case path.extname 859 when ".asc", ".sha512" 860 next 861 end 862 progress_reporter.increment_max 863 target_paths << path.to_s 864 end 865 target_paths.each do |path| 866 sign(path, "#{path}.asc") 867 sha512(path, "#{path}.sha512") 868 progress_reporter.advance 869 end 870 progress_reporter.finish 871 end 872 873 def download_distribution(distribution, 874 destination, 875 target, 876 list: nil, 877 pattern: nil, 878 prefix: nil) 879 mkdir_p(destination, verbose: verbose?) unless File.exist?(destination) 880 existing_paths = {} 881 Pathname(destination).glob("**/*") do |path| 882 next if path.directory? 883 existing_paths[path.to_s] = true 884 end 885 options = { 886 api_key: artifactory_api_key, 887 destination: destination, 888 distribution: distribution, 889 list: list, 890 pattern: pattern, 891 prefix: prefix, 892 staging: staging?, 893 } 894 options[:rc] = rc if target == :rc 895 downloader = ArtifactoryDownloader.new(**options) 896 downloader.download do |output_path| 897 existing_paths.delete(output_path) 898 end 899 existing_paths.each_key do |path| 900 rm_f(path, verbose: verbose?) 901 end 902 end 903 904 def same_content?(path1, path2) 905 File.exist?(path1) and 906 File.exist?(path2) and 907 Digest::SHA256.file(path1) == Digest::SHA256.file(path2) 908 end 909 910 def copy_artifact(source_path, 911 destination_path, 912 progress_reporter) 913 return if same_content?(source_path, destination_path) 914 progress_reporter.increment_max 915 destination_dir = File.dirname(destination_path) 916 unless File.exist?(destination_dir) 917 mkdir_p(destination_dir, verbose: verbose?) 918 end 919 cp(source_path, destination_path, verbose: verbose?) 920 progress_reporter.advance 921 end 922 923 def prepare_staging(base_path) 924 client = ArtifactoryClient.new(nil, artifactory_api_key) 925 ["", "-rc"].each do |suffix| 926 path = "#{base_path}#{suffix}" 927 progress_reporter = ProgressReporter.new("Preparing staging for #{path}") 928 progress_reporter.increment_max 929 begin 930 staging_path = "#{base_path}-staging#{suffix}" 931 if client.exist?(staging_path) 932 client.delete(staging_path) 933 end 934 if client.exist?(path) 935 client.copy(path, staging_path) 936 end 937 ensure 938 progress_reporter.advance 939 progress_reporter.finish 940 end 941 end 942 end 943 944 def delete_staging(base_path) 945 client = ArtifactoryClient.new(nil, artifactory_api_key) 946 ["", "-rc"].each do |suffix| 947 path = "#{base_path}#{suffix}" 948 progress_reporter = ProgressReporter.new("Deleting staging for #{path}") 949 progress_reporter.increment_max 950 begin 951 staging_path = "#{base_path}-staging#{suffix}" 952 if client.exist?(staging_path) 953 client.delete(staging_path) 954 end 955 ensure 956 progress_reporter.advance 957 progress_reporter.finish 958 end 959 end 960 end 961 962 def uploaded_files_name 963 "uploaded-files.txt" 964 end 965 966 def write_uploaded_files(dir) 967 dir = Pathname(dir) 968 uploaded_files = [] 969 dir.glob("**/*") do |path| 970 next if path.directory? 971 uploaded_files << path.relative_path_from(dir).to_s 972 end 973 File.open("#{dir}/#{uploaded_files_name}", "w") do |output| 974 output.puts(uploaded_files.sort) 975 end 976 end 977 978 def tmp_dir 979 "binary/tmp" 980 end 981 982 def rc_dir 983 "#{tmp_dir}/rc" 984 end 985 986 def release_dir 987 "#{tmp_dir}/release" 988 end 989 990 def apt_repository_label 991 "Apache Arrow" 992 end 993 994 def apt_repository_description 995 "Apache Arrow packages" 996 end 997 998 def apt_rc_repositories_dir 999 "#{rc_dir}/apt/repositories" 1000 end 1001 1002 def apt_release_repositories_dir 1003 "#{release_dir}/apt/repositories" 1004 end 1005 1006 def available_apt_targets 1007 [ 1008 ["debian", "buster", "main"], 1009 ["debian", "bullseye", "main"], 1010 ["debian", "bookworm", "main"], 1011 ["ubuntu", "bionic", "main"], 1012 ["ubuntu", "focal", "main"], 1013 ["ubuntu", "hirsute", "main"], 1014 ["ubuntu", "impish", "main"], 1015 ] 1016 end 1017 1018 def apt_targets 1019 env_apt_targets = (ENV["APT_TARGETS"] || "").split(",") 1020 if env_apt_targets.empty? 1021 available_apt_targets 1022 else 1023 available_apt_targets.select do |distribution, code_name, component| 1024 env_apt_targets.any? do |env_apt_target| 1025 if env_apt_target.include?("-") 1026 env_apt_target.start_with?("#{distribution}-#{code_name}") 1027 else 1028 env_apt_target == distribution 1029 end 1030 end 1031 end 1032 end 1033 end 1034 1035 def apt_distributions 1036 apt_targets.collect(&:first).uniq 1037 end 1038 1039 def apt_architectures 1040 [ 1041 "amd64", 1042 "arm64", 1043 ] 1044 end 1045 1046 def generate_apt_release(dists_dir, code_name, component, architecture) 1047 dir = "#{dists_dir}/#{component}/" 1048 if architecture == "source" 1049 dir << architecture 1050 else 1051 dir << "binary-#{architecture}" 1052 end 1053 1054 mkdir_p(dir, verbose: verbose?) 1055 File.open("#{dir}/Release", "w") do |release| 1056 release.puts(<<-RELEASE) 1057Archive: #{code_name} 1058Component: #{component} 1059Origin: #{apt_repository_label} 1060Label: #{apt_repository_label} 1061Architecture: #{architecture} 1062 RELEASE 1063 end 1064 end 1065 1066 def generate_apt_ftp_archive_generate_conf(code_name, component) 1067 conf = <<-CONF 1068Dir::ArchiveDir "."; 1069Dir::CacheDir "."; 1070TreeDefault::Directory "pool/#{code_name}/#{component}"; 1071TreeDefault::SrcDirectory "pool/#{code_name}/#{component}"; 1072Default::Packages::Extensions ".deb"; 1073Default::Packages::Compress ". gzip xz"; 1074Default::Sources::Compress ". gzip xz"; 1075Default::Contents::Compress "gzip"; 1076 CONF 1077 1078 apt_architectures.each do |architecture| 1079 conf << <<-CONF 1080 1081BinDirectory "dists/#{code_name}/#{component}/binary-#{architecture}" { 1082 Packages "dists/#{code_name}/#{component}/binary-#{architecture}/Packages"; 1083 Contents "dists/#{code_name}/#{component}/Contents-#{architecture}"; 1084 SrcPackages "dists/#{code_name}/#{component}/source/Sources"; 1085}; 1086 CONF 1087 end 1088 1089 conf << <<-CONF 1090 1091Tree "dists/#{code_name}" { 1092 Sections "#{component}"; 1093 Architectures "#{apt_architectures.join(" ")} source"; 1094}; 1095 CONF 1096 1097 conf 1098 end 1099 1100 def generate_apt_ftp_archive_release_conf(code_name, component) 1101 <<-CONF 1102APT::FTPArchive::Release::Origin "#{apt_repository_label}"; 1103APT::FTPArchive::Release::Label "#{apt_repository_label}"; 1104APT::FTPArchive::Release::Architectures "#{apt_architectures.join(" ")}"; 1105APT::FTPArchive::Release::Codename "#{code_name}"; 1106APT::FTPArchive::Release::Suite "#{code_name}"; 1107APT::FTPArchive::Release::Components "#{component}"; 1108APT::FTPArchive::Release::Description "#{apt_repository_description}"; 1109 CONF 1110 end 1111 1112 def apt_update(base_dir, incoming_dir, merged_dir) 1113 apt_targets.each do |distribution, code_name, component| 1114 distribution_dir = "#{incoming_dir}/#{distribution}" 1115 pool_dir = "#{distribution_dir}/pool/#{code_name}" 1116 next unless File.exist?(pool_dir) 1117 dists_dir = "#{distribution_dir}/dists/#{code_name}" 1118 rm_rf(dists_dir, verbose: verbose?) 1119 generate_apt_release(dists_dir, code_name, component, "source") 1120 apt_architectures.each do |architecture| 1121 generate_apt_release(dists_dir, code_name, component, architecture) 1122 end 1123 1124 generate_conf_file = Tempfile.new("apt-ftparchive-generate.conf") 1125 File.open(generate_conf_file.path, "w") do |conf| 1126 conf.puts(generate_apt_ftp_archive_generate_conf(code_name, 1127 component)) 1128 end 1129 cd(distribution_dir, verbose: verbose?) do 1130 sh("apt-ftparchive", 1131 "generate", 1132 generate_conf_file.path, 1133 out: default_output, 1134 verbose: verbose?) 1135 end 1136 1137 Dir.glob("#{dists_dir}/Release*") do |release| 1138 rm_f(release, verbose: verbose?) 1139 end 1140 Dir.glob("#{distribution_dir}/*.db") do |db| 1141 rm_f(db, verbose: verbose?) 1142 end 1143 release_conf_file = Tempfile.new("apt-ftparchive-release.conf") 1144 File.open(release_conf_file.path, "w") do |conf| 1145 conf.puts(generate_apt_ftp_archive_release_conf(code_name, 1146 component)) 1147 end 1148 release_file = Tempfile.new("apt-ftparchive-release") 1149 sh("apt-ftparchive", 1150 "-c", release_conf_file.path, 1151 "release", 1152 dists_dir, 1153 out: release_file.path, 1154 verbose: verbose?) 1155 mv(release_file.path, "#{dists_dir}/Release", verbose: verbose?) 1156 1157 base_dists_dir = "#{base_dir}/#{distribution}/dists/#{code_name}" 1158 merged_dists_dir = "#{merged_dir}/#{distribution}/dists/#{code_name}" 1159 rm_rf(merged_dists_dir) 1160 merger = APTDistsMerge::Merger.new(base_dists_dir, 1161 dists_dir, 1162 merged_dists_dir) 1163 merger.merge 1164 1165 in_release_path = "#{merged_dists_dir}/InRelease" 1166 release_path = "#{merged_dists_dir}/Release" 1167 signed_release_path = "#{release_path}.gpg" 1168 sh("gpg", 1169 "--sign", 1170 "--detach-sign", 1171 "--armor", 1172 "--local-user", gpg_key_id, 1173 "--output", signed_release_path, 1174 release_path, 1175 out: default_output, 1176 verbose: verbose?) 1177 sh("gpg", 1178 "--clear-sign", 1179 "--local-user", gpg_key_id, 1180 "--output", in_release_path, 1181 release_path, 1182 out: default_output, 1183 verbose: verbose?) 1184 end 1185 end 1186 1187 def define_apt_staging_tasks 1188 namespace :apt do 1189 namespace :staging do 1190 desc "Prepare staging environment for APT repositories" 1191 task :prepare do 1192 apt_distributions.each do |distribution| 1193 prepare_staging(distribution) 1194 end 1195 end 1196 1197 desc "Delete staging environment for APT repositories" 1198 task :delete do 1199 apt_distributions.each do |distribution| 1200 delete_staging(distribution) 1201 end 1202 end 1203 end 1204 end 1205 end 1206 1207 def define_apt_rc_tasks 1208 namespace :apt do 1209 namespace :rc do 1210 base_dir = "#{apt_rc_repositories_dir}/base" 1211 incoming_dir = "#{apt_rc_repositories_dir}/incoming" 1212 merged_dir = "#{apt_rc_repositories_dir}/merged" 1213 upload_dir = "#{apt_rc_repositories_dir}/upload" 1214 1215 desc "Copy .deb packages" 1216 task :copy do 1217 apt_targets.each do |distribution, code_name, component| 1218 progress_label = "Copying: #{distribution} #{code_name}" 1219 progress_reporter = ProgressReporter.new(progress_label) 1220 1221 distribution_dir = "#{incoming_dir}/#{distribution}" 1222 pool_dir = "#{distribution_dir}/pool/#{code_name}" 1223 rm_rf(pool_dir, verbose: verbose?) 1224 mkdir_p(pool_dir, verbose: verbose?) 1225 source_dir_prefix = "#{artifacts_dir}/#{distribution}-#{code_name}" 1226 Dir.glob("#{source_dir_prefix}*/**/*") do |path| 1227 next if File.directory?(path) 1228 base_name = File.basename(path) 1229 if base_name.start_with?("apache-arrow-apt-source") 1230 package_name = "apache-arrow-apt-source" 1231 else 1232 package_name = "apache-arrow" 1233 end 1234 destination_path = [ 1235 pool_dir, 1236 component, 1237 package_name[0], 1238 package_name, 1239 base_name, 1240 ].join("/") 1241 copy_artifact(path, 1242 destination_path, 1243 progress_reporter) 1244 case base_name 1245 when /\A[^_]+-apt-source_.*\.deb\z/ 1246 latest_apt_source_package_path = [ 1247 distribution_dir, 1248 "#{package_name}-latest-#{code_name}.deb" 1249 ].join("/") 1250 copy_artifact(path, 1251 latest_apt_source_package_path, 1252 progress_reporter) 1253 end 1254 end 1255 progress_reporter.finish 1256 end 1257 end 1258 1259 desc "Download dists/ for RC APT repositories" 1260 task :download do 1261 apt_distributions.each do |distribution| 1262 not_checksum_pattern = /.+(?<!\.asc|\.sha512)\z/ 1263 base_distribution_dir = "#{base_dir}/#{distribution}" 1264 pattern = /\Adists\/#{not_checksum_pattern}/ 1265 download_distribution(distribution, 1266 base_distribution_dir, 1267 :base, 1268 pattern: pattern) 1269 end 1270 end 1271 1272 desc "Sign .deb packages" 1273 task :sign do 1274 apt_distributions.each do |distribution| 1275 distribution_dir = "#{incoming_dir}/#{distribution}" 1276 Dir.glob("#{distribution_dir}/**/*.dsc") do |path| 1277 begin 1278 sh({"LANG" => "C"}, 1279 "gpg", 1280 "--verify", 1281 path, 1282 out: IO::NULL, 1283 err: IO::NULL, 1284 verbose: false) 1285 rescue 1286 sh("debsign", 1287 "--no-re-sign", 1288 "-k#{gpg_key_id}", 1289 path, 1290 out: default_output, 1291 verbose: verbose?) 1292 end 1293 end 1294 sign_dir(distribution, distribution_dir) 1295 end 1296 end 1297 1298 desc "Update RC APT repositories" 1299 task :update do 1300 apt_update(base_dir, incoming_dir, merged_dir) 1301 apt_targets.each do |distribution, code_name, component| 1302 dists_dir = "#{merged_dir}/#{distribution}/dists/#{code_name}" 1303 next unless File.exist?(dists_dir) 1304 sign_dir("#{distribution} #{code_name}", 1305 dists_dir) 1306 end 1307 end 1308 1309 desc "Upload .deb packages and RC APT repositories" 1310 task :upload do 1311 apt_distributions.each do |distribution| 1312 upload_distribution_dir = "#{upload_dir}/#{distribution}" 1313 incoming_distribution_dir = "#{incoming_dir}/#{distribution}" 1314 merged_dists_dir = "#{merged_dir}/#{distribution}/dists" 1315 1316 rm_rf(upload_distribution_dir, verbose: verbose?) 1317 mkdir_p(upload_distribution_dir, verbose: verbose?) 1318 Dir.glob("#{incoming_distribution_dir}/*") do |path| 1319 next if File.basename(path) == "dists" 1320 cp_r(path, 1321 upload_distribution_dir, 1322 preserve: true, 1323 verbose: verbose?) 1324 end 1325 cp_r(merged_dists_dir, 1326 upload_distribution_dir, 1327 preserve: true, 1328 verbose: verbose?) 1329 write_uploaded_files(upload_distribution_dir) 1330 uploader = ArtifactoryUploader.new(api_key: artifactory_api_key, 1331 distribution: distribution, 1332 rc: rc, 1333 source: upload_distribution_dir, 1334 staging: staging?) 1335 uploader.upload 1336 end 1337 end 1338 end 1339 1340 desc "Release RC APT repositories" 1341 apt_rc_tasks = [ 1342 "apt:rc:copy", 1343 "apt:rc:download", 1344 "apt:rc:sign", 1345 "apt:rc:update", 1346 "apt:rc:upload", 1347 ] 1348 apt_rc_tasks.unshift("apt:staging:prepare") if staging? 1349 task :rc => apt_rc_tasks 1350 end 1351 end 1352 1353 def define_apt_release_tasks 1354 directory apt_release_repositories_dir 1355 1356 namespace :apt do 1357 namespace :release do 1358 desc "Download RC APT repositories" 1359 task :download => apt_release_repositories_dir do 1360 apt_distributions.each do |distribution| 1361 distribution_dir = "#{apt_release_repositories_dir}/#{distribution}" 1362 download_distribution(distribution, 1363 distribution_dir, 1364 :rc, 1365 list: uploaded_files_name) 1366 end 1367 end 1368 1369 desc "Upload release APT repositories" 1370 task :upload => apt_release_repositories_dir do 1371 apt_distributions.each do |distribution| 1372 distribution_dir = "#{apt_release_repositories_dir}/#{distribution}" 1373 uploader = ArtifactoryUploader.new(api_key: artifactory_api_key, 1374 distribution: distribution, 1375 source: distribution_dir, 1376 staging: staging?) 1377 uploader.upload 1378 end 1379 end 1380 end 1381 1382 desc "Release APT repositories" 1383 apt_release_tasks = [ 1384 "apt:release:download", 1385 "apt:release:upload", 1386 ] 1387 task :release => apt_release_tasks 1388 end 1389 end 1390 1391 def define_apt_tasks 1392 define_apt_staging_tasks 1393 define_apt_rc_tasks 1394 define_apt_release_tasks 1395 end 1396 1397 def yum_rc_repositories_dir 1398 "#{rc_dir}/yum/repositories" 1399 end 1400 1401 def yum_release_repositories_dir 1402 "#{release_dir}/yum/repositories" 1403 end 1404 1405 def available_yum_targets 1406 [ 1407 ["almalinux", "8"], 1408 ["amazon-linux", "2"], 1409 ["centos", "7"], 1410 ["centos", "8"], 1411 ] 1412 end 1413 1414 def yum_targets 1415 env_yum_targets = (ENV["YUM_TARGETS"] || "").split(",") 1416 if env_yum_targets.empty? 1417 available_yum_targets 1418 else 1419 available_yum_targets.select do |distribution, distribution_version| 1420 env_yum_targets.any? do |env_yum_target| 1421 if /\d/.match?(env_yum_target) 1422 env_yum_target.start_with?("#{distribution}-#{distribution_version}") 1423 else 1424 env_yum_target == distribution 1425 end 1426 end 1427 end 1428 end 1429 end 1430 1431 def yum_distributions 1432 yum_targets.collect(&:first).uniq 1433 end 1434 1435 def yum_architectures 1436 [ 1437 "aarch64", 1438 "x86_64", 1439 ] 1440 end 1441 1442 def signed_rpm?(rpm) 1443 IO.pipe do |input, output| 1444 system("rpm", "--checksig", rpm, out: output) 1445 output.close 1446 signature = input.gets.sub(/\A#{Regexp.escape(rpm)}: /, "") 1447 signature.split.include?("signatures") 1448 end 1449 end 1450 1451 def sign_rpms(directory) 1452 thread_pool = ThreadPool.new(:gpg) do |rpm| 1453 unless signed_rpm?(rpm) 1454 sh("rpm", 1455 "-D", "_gpg_name #{gpg_key_id}", 1456 "-D", "__gpg /usr/bin/gpg", 1457 "-D", "__gpg_check_password_cmd /bin/true true", 1458 "--resign", 1459 rpm, 1460 out: default_output, 1461 verbose: verbose?) 1462 end 1463 end 1464 Dir.glob("#{directory}/**/*.rpm") do |rpm| 1465 thread_pool << rpm 1466 end 1467 thread_pool.join 1468 end 1469 1470 def rpm_sign(directory) 1471 unless system("rpm", "-q", 1472 rpm_gpg_key_package_name(gpg_key_id), 1473 out: IO::NULL) 1474 gpg_key = Tempfile.new(["apache-arrow-binary", ".asc"]) 1475 sh("gpg", 1476 "--armor", 1477 "--export", gpg_key_id, 1478 out: gpg_key.path, 1479 verbose: verbose?) 1480 sh("rpm", 1481 "--import", gpg_key.path, 1482 out: default_output, 1483 verbose: verbose?) 1484 gpg_key.close! 1485 end 1486 1487 yum_targets.each do |distribution, distribution_version| 1488 source_dir = [ 1489 directory, 1490 distribution, 1491 distribution_version, 1492 ].join("/") 1493 sign_rpms(source_dir) 1494 end 1495 end 1496 1497 def yum_update(base_dir, incoming_dir) 1498 yum_targets.each do |distribution, distribution_version| 1499 target_dir = "#{incoming_dir}/#{distribution}/#{distribution_version}" 1500 target_dir = Pathname(target_dir) 1501 next unless target_dir.directory? 1502 Dir.glob("#{target_dir}/**/repodata") do |repodata| 1503 rm_rf(repodata, verbose: verbose?) 1504 end 1505 target_dir.glob("*") do |arch_dir| 1506 next unless arch_dir.directory? 1507 base_repodata_dir = [ 1508 base_dir, 1509 distribution, 1510 distribution_version, 1511 File.basename(arch_dir), 1512 "repodata", 1513 ].join("/") 1514 if File.exist?(base_repodata_dir) 1515 cp_r(base_repodata_dir, 1516 arch_dir.to_s, 1517 preserve: true, 1518 verbose: verbose?) 1519 end 1520 packages = Tempfile.new("createrepo-c-packages") 1521 Pathname.glob("#{arch_dir}/*/*.rpm") do |rpm| 1522 relative_rpm = rpm.relative_path_from(arch_dir) 1523 packages.puts(relative_rpm.to_s) 1524 end 1525 packages.close 1526 sh("createrepo_c", 1527 "--pkglist", packages.path, 1528 "--recycle-pkglist", 1529 "--retain-old-md-by-age=0", 1530 "--skip-stat", 1531 "--update", 1532 arch_dir.to_s, 1533 out: default_output, 1534 verbose: verbose?) 1535 end 1536 end 1537 end 1538 1539 def define_yum_staging_tasks 1540 namespace :yum do 1541 namespace :staging do 1542 desc "Prepare staging environment for Yum repositories" 1543 task :prepare do 1544 yum_distributions.each do |distribution| 1545 prepare_staging(distribution) 1546 end 1547 end 1548 1549 desc "Delete staging environment for Yum repositories" 1550 task :delete do 1551 yum_distributions.each do |distribution| 1552 delete_staging(distribution) 1553 end 1554 end 1555 end 1556 end 1557 end 1558 1559 def define_yum_rc_tasks 1560 namespace :yum do 1561 namespace :rc do 1562 base_dir = "#{yum_rc_repositories_dir}/base" 1563 incoming_dir = "#{yum_rc_repositories_dir}/incoming" 1564 upload_dir = "#{yum_rc_repositories_dir}/upload" 1565 1566 desc "Copy RPM packages" 1567 task :copy do 1568 yum_targets.each do |distribution, distribution_version| 1569 progress_label = "Copying: #{distribution} #{distribution_version}" 1570 progress_reporter = ProgressReporter.new(progress_label) 1571 1572 destination_prefix = [ 1573 incoming_dir, 1574 distribution, 1575 distribution_version, 1576 ].join("/") 1577 rm_rf(destination_prefix, verbose: verbose?) 1578 source_dir_prefix = 1579 "#{artifacts_dir}/#{distribution}-#{distribution_version}" 1580 Dir.glob("#{source_dir_prefix}*/**/*") do |path| 1581 next if File.directory?(path) 1582 base_name = File.basename(path) 1583 type = base_name.split(".")[-2] 1584 destination_paths = [] 1585 case type 1586 when "src" 1587 destination_paths << [ 1588 destination_prefix, 1589 "Source", 1590 "SPackages", 1591 base_name, 1592 ].join("/") 1593 when "noarch" 1594 yum_architectures.each do |architecture| 1595 destination_paths << [ 1596 destination_prefix, 1597 architecture, 1598 "Packages", 1599 base_name, 1600 ].join("/") 1601 end 1602 else 1603 destination_paths << [ 1604 destination_prefix, 1605 type, 1606 "Packages", 1607 base_name, 1608 ].join("/") 1609 end 1610 destination_paths.each do |destination_path| 1611 copy_artifact(path, 1612 destination_path, 1613 progress_reporter) 1614 end 1615 case base_name 1616 when /\A(apache-arrow-release)-.*\.noarch\.rpm\z/ 1617 package_name = $1 1618 latest_release_package_path = [ 1619 destination_prefix, 1620 "#{package_name}-latest.rpm" 1621 ].join("/") 1622 copy_artifact(path, 1623 latest_release_package_path, 1624 progress_reporter) 1625 end 1626 end 1627 1628 progress_reporter.finish 1629 end 1630 end 1631 1632 desc "Download repodata for RC Yum repositories" 1633 task :download do 1634 yum_distributions.each do |distribution| 1635 distribution_dir = "#{base_dir}/#{distribution}" 1636 download_distribution(distribution, 1637 distribution_dir, 1638 :base, 1639 pattern: /\/repodata\//) 1640 end 1641 end 1642 1643 desc "Sign RPM packages" 1644 task :sign do 1645 rpm_sign(incoming_dir) 1646 yum_targets.each do |distribution, distribution_version| 1647 source_dir = [ 1648 incoming_dir, 1649 distribution, 1650 distribution_version, 1651 ].join("/") 1652 sign_dir("#{distribution}-#{distribution_version}", 1653 source_dir) 1654 end 1655 end 1656 1657 desc "Update RC Yum repositories" 1658 task :update do 1659 yum_update(base_dir, incoming_dir) 1660 yum_targets.each do |distribution, distribution_version| 1661 target_dir = [ 1662 incoming_dir, 1663 distribution, 1664 distribution_version, 1665 ].join("/") 1666 target_dir = Pathname(target_dir) 1667 next unless target_dir.directory? 1668 target_dir.glob("*") do |arch_dir| 1669 next unless arch_dir.directory? 1670 sign_label = 1671 "#{distribution}-#{distribution_version} #{arch_dir.basename}" 1672 sign_dir(sign_label, 1673 arch_dir.to_s) 1674 end 1675 end 1676 end 1677 1678 desc "Upload RC Yum repositories" 1679 task :upload => yum_rc_repositories_dir do 1680 yum_distributions.each do |distribution| 1681 incoming_target_dir = "#{incoming_dir}/#{distribution}" 1682 upload_target_dir = "#{upload_dir}/#{distribution}" 1683 1684 rm_rf(upload_target_dir, verbose: verbose?) 1685 mkdir_p(upload_target_dir, verbose: verbose?) 1686 cp_r(Dir.glob("#{incoming_target_dir}/*"), 1687 upload_target_dir.to_s, 1688 preserve: true, 1689 verbose: verbose?) 1690 write_uploaded_files(upload_target_dir) 1691 1692 uploader = ArtifactoryUploader.new(api_key: artifactory_api_key, 1693 distribution: distribution, 1694 rc: rc, 1695 source: upload_target_dir, 1696 staging: staging?, 1697 sync: true, 1698 sync_pattern: /\/repodata\//) 1699 uploader.upload 1700 end 1701 end 1702 end 1703 1704 desc "Release RC Yum packages" 1705 yum_rc_tasks = [ 1706 "yum:rc:copy", 1707 "yum:rc:download", 1708 "yum:rc:sign", 1709 "yum:rc:update", 1710 "yum:rc:upload", 1711 ] 1712 yum_rc_tasks.unshift("yum:staging:prepare") if staging? 1713 task :rc => yum_rc_tasks 1714 end 1715 end 1716 1717 def define_yum_release_tasks 1718 directory yum_release_repositories_dir 1719 1720 namespace :yum do 1721 namespace :release do 1722 desc "Download RC Yum repositories" 1723 task :download => yum_release_repositories_dir do 1724 yum_distributions.each do |distribution| 1725 distribution_dir = "#{yum_release_repositories_dir}/#{distribution}" 1726 download_distribution(distribution, 1727 distribution_dir, 1728 :rc, 1729 list: uploaded_files_name) 1730 end 1731 end 1732 1733 desc "Upload release Yum repositories" 1734 task :upload => yum_release_repositories_dir do 1735 yum_distributions.each do |distribution| 1736 distribution_dir = "#{yum_release_repositories_dir}/#{distribution}" 1737 uploader = 1738 ArtifactoryUploader.new(api_key: artifactory_api_key, 1739 distribution: distribution, 1740 source: distribution_dir, 1741 staging: staging?, 1742 sync: true, 1743 sync_pattern: /\/repodata\//) 1744 uploader.upload 1745 end 1746 end 1747 end 1748 1749 desc "Release Yum packages" 1750 yum_release_tasks = [ 1751 "yum:release:download", 1752 "yum:release:upload", 1753 ] 1754 task :release => yum_release_tasks 1755 end 1756 end 1757 1758 def define_yum_tasks 1759 define_yum_staging_tasks 1760 define_yum_rc_tasks 1761 define_yum_release_tasks 1762 end 1763 1764 def define_generic_data_rc_tasks(label, 1765 id, 1766 rc_dir, 1767 target_files_glob) 1768 directory rc_dir 1769 1770 namespace id do 1771 namespace :rc do 1772 desc "Copy #{label} packages" 1773 task :copy => rc_dir do 1774 progress_label = "Copying: #{label}" 1775 progress_reporter = ProgressReporter.new(progress_label) 1776 1777 Pathname(artifacts_dir).glob(target_files_glob) do |path| 1778 next if path.directory? 1779 destination_path = [ 1780 rc_dir, 1781 path.basename.to_s, 1782 ].join("/") 1783 copy_artifact(path, destination_path, progress_reporter) 1784 end 1785 1786 progress_reporter.finish 1787 end 1788 1789 desc "Sign #{label} packages" 1790 task :sign => rc_dir do 1791 sign_dir(label, rc_dir) 1792 end 1793 1794 desc "Upload #{label} packages" 1795 task :upload do 1796 uploader = 1797 ArtifactoryUploader.new(api_key: artifactory_api_key, 1798 destination_prefix: full_version, 1799 distribution: id.to_s, 1800 rc: rc, 1801 source: rc_dir, 1802 staging: staging?) 1803 uploader.upload 1804 end 1805 end 1806 1807 desc "Release RC #{label} packages" 1808 rc_tasks = [ 1809 "#{id}:rc:copy", 1810 "#{id}:rc:sign", 1811 "#{id}:rc:upload", 1812 ] 1813 task :rc => rc_tasks 1814 end 1815 end 1816 1817 def define_generic_data_release_tasks(label, id, release_dir) 1818 directory release_dir 1819 1820 namespace id do 1821 namespace :release do 1822 desc "Download RC #{label} packages" 1823 task :download => release_dir do 1824 download_distribution(id.to_s, 1825 release_dir, 1826 :rc, 1827 prefix: "#{full_version}") 1828 end 1829 1830 desc "Upload release #{label} packages" 1831 task :upload => release_dir do 1832 uploader = ArtifactoryUploader.new(api_key: artifactory_api_key, 1833 destination_prefix: version, 1834 distribution: id.to_s, 1835 source: release_dir, 1836 staging: staging?) 1837 uploader.upload 1838 end 1839 end 1840 1841 desc "Release #{label} packages" 1842 release_tasks = [ 1843 "#{id}:release:download", 1844 "#{id}:release:upload", 1845 ] 1846 task :release => release_tasks 1847 end 1848 end 1849 1850 def define_generic_data_tasks(label, 1851 id, 1852 rc_dir, 1853 release_dir, 1854 target_files_glob) 1855 define_generic_data_rc_tasks(label, id, rc_dir, target_files_glob) 1856 define_generic_data_release_tasks(label, id, release_dir) 1857 end 1858 1859 def define_python_tasks 1860 define_generic_data_tasks("Python", 1861 :python, 1862 "#{rc_dir}/python/#{full_version}", 1863 "#{release_dir}/python/#{full_version}", 1864 "{python-sdist,wheel-*}/**/*") 1865 end 1866 1867 def define_nuget_tasks 1868 define_generic_data_tasks("NuGet", 1869 :nuget, 1870 "#{rc_dir}/nuget/#{full_version}", 1871 "#{release_dir}/nuget/#{full_version}", 1872 "nuget/**/*") 1873 end 1874 1875 def define_summary_tasks 1876 namespace :summary do 1877 desc "Show RC summary" 1878 task :rc do 1879 suffix = "" 1880 suffix << "-staging" if staging? 1881 puts(<<-SUMMARY) 1882Success! The release candidate binaries are available here: 1883 https://apache.jfrog.io/artifactory/arrow/almalinux#{suffix}-rc/ 1884 https://apache.jfrog.io/artifactory/arrow/amazon-linux#{suffix}-rc/ 1885 https://apache.jfrog.io/artifactory/arrow/centos#{suffix}-rc/ 1886 https://apache.jfrog.io/artifactory/arrow/debian#{suffix}-rc/ 1887 https://apache.jfrog.io/artifactory/arrow/nuget#{suffix}-rc/#{full_version} 1888 https://apache.jfrog.io/artifactory/arrow/python#{suffix}-rc/#{full_version} 1889 https://apache.jfrog.io/artifactory/arrow/ubuntu#{suffix}-rc/ 1890 SUMMARY 1891 end 1892 1893 desc "Show release summary" 1894 task :release do 1895 suffix = "" 1896 suffix << "-staging" if staging? 1897 puts(<<-SUMMARY) 1898Success! The release binaries are available here: 1899 https://apache.jfrog.io/artifactory/arrow/almalinux#{suffix}/ 1900 https://apache.jfrog.io/artifactory/arrow/amazon-linux#{suffix}/ 1901 https://apache.jfrog.io/artifactory/arrow/centos#{suffix}/ 1902 https://apache.jfrog.io/artifactory/arrow/debian#{suffix}/ 1903 https://apache.jfrog.io/artifactory/arrow/nuget#{suffix}/#{version} 1904 https://apache.jfrog.io/artifactory/arrow/python#{suffix}/#{version} 1905 https://apache.jfrog.io/artifactory/arrow/ubuntu#{suffix}/ 1906 SUMMARY 1907 end 1908 end 1909 end 1910end 1911