1# Licensed to the Apache Software Foundation (ASF) under one 2# or more contributor license agreements. See the NOTICE file 3# distributed with this work for additional information 4# regarding copyright ownership. The ASF licenses this file 5# to you under the Apache License, Version 2.0 (the 6# "License"); you may not use this file except in compliance 7# with the License. You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, 12# software distributed under the License is distributed on an 13# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14# KIND, either express or implied. See the License for the 15# specific language governing permissions and limitations 16# under the License. 17 18require "English" 19require "open-uri" 20require "time" 21 22class PackageTask 23 include Rake::DSL 24 25 def initialize(package, version, release_time, options={}) 26 @package = package 27 @version = version 28 @release_time = release_time 29 30 @archive_base_name = "#{@package}-#{@version}" 31 @archive_name = "#{@archive_base_name}.tar.gz" 32 @full_archive_name = File.expand_path(@archive_name) 33 34 @rpm_package = @package 35 case @version 36 when /-((dev|rc)\d+)\z/ 37 base_version = $PREMATCH 38 sub_version = $1 39 type = $2 40 if type == "rc" and options[:rc_build_type] == :release 41 @deb_upstream_version = base_version 42 @deb_archive_base_name_version = base_version 43 @rpm_version = base_version 44 @rpm_release = "1" 45 else 46 @deb_upstream_version = "#{base_version}~#{sub_version}" 47 @deb_archive_base_name_version = @version 48 @rpm_version = base_version 49 @rpm_release = "0.#{sub_version}" 50 end 51 else 52 @deb_upstream_version = @version 53 @deb_archive_base_name_version = @version 54 @rpm_version = @version 55 @rpm_release = "1" 56 end 57 @deb_release = ENV["DEB_RELEASE"] || "1" 58 end 59 60 def define 61 define_dist_task 62 define_apt_task 63 define_yum_task 64 define_version_task 65 define_docker_tasks 66 end 67 68 private 69 def env_value(name) 70 value = ENV[name] 71 raise "Specify #{name} environment variable" if value.nil? 72 value 73 end 74 75 def debug_build? 76 ENV["DEBUG"] != "no" 77 end 78 79 def git_directory?(directory) 80 candidate_paths = [".git", "HEAD"] 81 candidate_paths.any? do |candidate_path| 82 File.exist?(File.join(directory, candidate_path)) 83 end 84 end 85 86 def latest_commit_time(git_directory) 87 return nil unless git_directory?(git_directory) 88 cd(git_directory) do 89 return Time.iso8601(`git log -n 1 --format=%aI`.chomp).utc 90 end 91 end 92 93 def download(url, output_path) 94 if File.directory?(output_path) 95 base_name = url.split("/").last 96 output_path = File.join(output_path, base_name) 97 end 98 absolute_output_path = File.expand_path(output_path) 99 100 unless File.exist?(absolute_output_path) 101 mkdir_p(File.dirname(absolute_output_path)) 102 rake_output_message "Downloading... #{url}" 103 open_url(url) do |downloaded_file| 104 File.open(absolute_output_path, "wb") do |output_file| 105 IO.copy_stream(downloaded_file, output_file) 106 end 107 end 108 end 109 110 absolute_output_path 111 end 112 113 def open_url(url, &block) 114 URI(url).open(&block) 115 end 116 117 def substitute_content(content) 118 content.gsub(/@(.+?)@/) do |matched| 119 yield($1, matched) 120 end 121 end 122 123 def docker_image(os, architecture) 124 image = "#{@package}-#{os}" 125 image << "-#{architecture}" if architecture 126 image 127 end 128 129 def docker_run(os, architecture, console: false) 130 id = os 131 id = "#{id}-#{architecture}" if architecture 132 image = docker_image(os, architecture) 133 build_command_line = [ 134 "docker", 135 "build", 136 "--cache-from", image, 137 "--tag", image, 138 ] 139 run_command_line = [ 140 "docker", 141 "run", 142 "--rm", 143 "--log-driver", "none", 144 "--volume", "#{Dir.pwd}:/host:rw", 145 ] 146 if $stdin.tty? 147 run_command_line << "--interactive" 148 run_command_line << "--tty" 149 else 150 run_command_line.concat(["--attach", "STDOUT"]) 151 run_command_line.concat(["--attach", "STDERR"]) 152 end 153 build_dir = ENV["BUILD_DIR"] 154 if build_dir 155 build_dir = "#{File.expand_path(build_dir)}/#{id}" 156 mkdir_p(build_dir) 157 run_command_line.concat(["--volume", "#{build_dir}:/build:rw"]) 158 end 159 if debug_build? 160 build_command_line.concat(["--build-arg", "DEBUG=yes"]) 161 run_command_line.concat(["--env", "DEBUG=yes"]) 162 end 163 pass_through_env_names = [ 164 "DEB_BUILD_OPTIONS", 165 "RPM_BUILD_NCPUS", 166 ] 167 pass_through_env_names.each do |name| 168 value = ENV[name] 169 next unless value 170 run_command_line.concat(["--env", "#{name}=#{value}"]) 171 end 172 if File.exist?(File.join(id, "Dockerfile")) 173 docker_context = id 174 else 175 from = File.readlines(File.join(id, "from")).find do |line| 176 /^[a-z]/i =~ line 177 end 178 build_command_line.concat(["--build-arg", "FROM=#{from.chomp}"]) 179 docker_context = os 180 end 181 build_command_line.concat(docker_build_options(os, architecture)) 182 run_command_line.concat(docker_run_options(os, architecture)) 183 build_command_line << docker_context 184 run_command_line << image 185 run_command_line << "/host/build.sh" unless console 186 187 sh(*build_command_line) 188 sh(*run_command_line) 189 end 190 191 def docker_build_options(os, architecture) 192 [] 193 end 194 195 def docker_run_options(os, architecture) 196 [] 197 end 198 199 def docker_pull(os, architecture) 200 image = docker_image(os, architecture) 201 command_line = [ 202 "docker", 203 "pull", 204 image, 205 ] 206 command_line.concat(docker_pull_options(os, architecture)) 207 sh(*command_line) 208 end 209 210 def docker_pull_options(os, architecture) 211 [] 212 end 213 214 def docker_push(os, architecture) 215 image = docker_image(os, architecture) 216 command_line = [ 217 "docker", 218 "push", 219 image, 220 ] 221 command_line.concat(docker_push_options(os, architecture)) 222 sh(*command_line) 223 end 224 225 def docker_push_options(os, architecture) 226 [] 227 end 228 229 def define_dist_task 230 define_archive_task 231 desc "Create release package" 232 task :dist => [@archive_name] 233 end 234 235 def split_target(target) 236 components = target.split("-") 237 if components[0, 2] == ["amazon", "linux"] 238 components[0, 2] = components[0, 2].join("-") 239 end 240 if components.size >= 3 241 components[2..-1] = components[2..-1].join("-") 242 end 243 components 244 end 245 246 def enable_apt? 247 true 248 end 249 250 def apt_targets 251 return [] unless enable_apt? 252 253 targets = (ENV["APT_TARGETS"] || "").split(",") 254 targets = apt_targets_default if targets.empty? 255 256 targets.find_all do |target| 257 Dir.exist?(File.join(apt_dir, target)) 258 end 259 end 260 261 def apt_targets_default 262 # Disable arm64 targets by default for now 263 # because they require some setups on host. 264 [ 265 "debian-buster", 266 # "debian-buster-arm64", 267 "debian-bullseye", 268 # "debian-bullseye-arm64", 269 "debian-bookworm", 270 # "debian-bookworm-arm64", 271 "ubuntu-bionic", 272 # "ubuntu-bionic-arm64", 273 "ubuntu-focal", 274 # "ubuntu-focal-arm64", 275 "ubuntu-hirsute", 276 # "ubuntu-hirsute-arm64", 277 "ubuntu-impish", 278 # "ubuntu-impish-arm64", 279 ] 280 end 281 282 def deb_archive_base_name 283 "#{@package}-#{@deb_archive_base_name_version}" 284 end 285 286 def deb_archive_name 287 "#{@package}-#{@deb_upstream_version}.tar.gz" 288 end 289 290 def apt_dir 291 "apt" 292 end 293 294 def apt_prepare_debian_dir(tmp_dir, target) 295 source_debian_dir = nil 296 specific_debian_dir = "debian.#{target}" 297 distribution, code_name, _architecture = split_target(target) 298 platform = [distribution, code_name].join("-") 299 platform_debian_dir = "debian.#{platform}" 300 if File.exist?(specific_debian_dir) 301 source_debian_dir = specific_debian_dir 302 elsif File.exist?(platform_debian_dir) 303 source_debian_dir = platform_debian_dir 304 else 305 source_debian_dir = "debian" 306 end 307 308 prepared_debian_dir = "#{tmp_dir}/debian.#{target}" 309 cp_r(source_debian_dir, prepared_debian_dir) 310 control_in_path = "#{prepared_debian_dir}/control.in" 311 if File.exist?(control_in_path) 312 control_in = File.read(control_in_path) 313 rm_f(control_in_path) 314 File.open("#{prepared_debian_dir}/control", "w") do |control| 315 prepared_control = apt_prepare_debian_control(control_in, target) 316 control.print(prepared_control) 317 end 318 end 319 end 320 321 def apt_prepare_debian_control(control_in, target) 322 message = "#{__method__} must be defined to use debian/control.in" 323 raise NotImplementedError, message 324 end 325 326 def apt_build(console: false) 327 tmp_dir = "#{apt_dir}/tmp" 328 rm_rf(tmp_dir) 329 mkdir_p(tmp_dir) 330 cp(deb_archive_name, 331 File.join(tmp_dir, deb_archive_name)) 332 apt_targets.each do |target| 333 apt_prepare_debian_dir(tmp_dir, target) 334 end 335 336 env_sh = "#{apt_dir}/env.sh" 337 File.open(env_sh, "w") do |file| 338 file.puts(<<-ENV) 339PACKAGE=#{@package} 340VERSION=#{@deb_upstream_version} 341 ENV 342 end 343 344 apt_targets.each do |target| 345 cd(apt_dir) do 346 distribution, version, architecture = split_target(target) 347 os = "#{distribution}-#{version}" 348 docker_run(os, architecture, console: console) 349 end 350 end 351 end 352 353 def define_apt_task 354 namespace :apt do 355 source_build_sh = "#{__dir__}/apt/build.sh" 356 build_sh = "#{apt_dir}/build.sh" 357 repositories_dir = "#{apt_dir}/repositories" 358 359 file build_sh => source_build_sh do 360 cp(source_build_sh, build_sh) 361 end 362 363 directory repositories_dir 364 365 desc "Build deb packages" 366 if enable_apt? 367 build_dependencies = [ 368 deb_archive_name, 369 build_sh, 370 repositories_dir, 371 ] 372 else 373 build_dependencies = [] 374 end 375 task :build => build_dependencies do 376 apt_build if enable_apt? 377 end 378 379 namespace :build do 380 desc "Open console" 381 task :console => build_dependencies do 382 apt_build(console: true) if enable_apt? 383 end 384 end 385 end 386 387 desc "Release APT repositories" 388 apt_tasks = [ 389 "apt:build", 390 ] 391 task :apt => apt_tasks 392 end 393 394 def enable_yum? 395 true 396 end 397 398 def yum_targets 399 return [] unless enable_yum? 400 401 targets = (ENV["YUM_TARGETS"] || "").split(",") 402 targets = yum_targets_default if targets.empty? 403 404 targets.find_all do |target| 405 Dir.exist?(File.join(yum_dir, target)) 406 end 407 end 408 409 def yum_targets_default 410 # Disable aarch64 targets by default for now 411 # because they require some setups on host. 412 [ 413 "almalinux-8", 414 # "almalinux-8-arch64", 415 "amazon-linux-2", 416 # "amazon-linux-2-arch64", 417 "centos-7", 418 # "centos-7-aarch64", 419 "centos-8", 420 # "centos-8-aarch64", 421 ] 422 end 423 424 def rpm_archive_base_name 425 "#{@package}-#{@rpm_version}" 426 end 427 428 def rpm_archive_name 429 "#{rpm_archive_base_name}.tar.gz" 430 end 431 432 def yum_dir 433 "yum" 434 end 435 436 def yum_build_sh 437 "#{yum_dir}/build.sh" 438 end 439 440 def yum_expand_variable(key) 441 case key 442 when "PACKAGE" 443 @rpm_package 444 when "VERSION" 445 @rpm_version 446 when "RELEASE" 447 @rpm_release 448 else 449 nil 450 end 451 end 452 453 def yum_spec_in_path 454 "#{yum_dir}/#{@rpm_package}.spec.in" 455 end 456 457 def yum_build(console: false) 458 tmp_dir = "#{yum_dir}/tmp" 459 rm_rf(tmp_dir) 460 mkdir_p(tmp_dir) 461 cp(rpm_archive_name, 462 File.join(tmp_dir, rpm_archive_name)) 463 464 env_sh = "#{yum_dir}/env.sh" 465 File.open(env_sh, "w") do |file| 466 file.puts(<<-ENV) 467SOURCE_ARCHIVE=#{rpm_archive_name} 468PACKAGE=#{@rpm_package} 469VERSION=#{@rpm_version} 470RELEASE=#{@rpm_release} 471 ENV 472 end 473 474 spec = "#{tmp_dir}/#{@rpm_package}.spec" 475 spec_in_data = File.read(yum_spec_in_path) 476 spec_data = substitute_content(spec_in_data) do |key, matched| 477 yum_expand_variable(key) || matched 478 end 479 File.open(spec, "w") do |spec_file| 480 spec_file.print(spec_data) 481 end 482 483 yum_targets.each do |target| 484 cd(yum_dir) do 485 distribution, version, architecture = split_target(target) 486 os = "#{distribution}-#{version}" 487 docker_run(os, architecture, console: console) 488 end 489 end 490 end 491 492 def define_yum_task 493 namespace :yum do 494 source_build_sh = "#{__dir__}/yum/build.sh" 495 file yum_build_sh => source_build_sh do 496 cp(source_build_sh, yum_build_sh) 497 end 498 499 repositories_dir = "#{yum_dir}/repositories" 500 directory repositories_dir 501 502 desc "Build RPM packages" 503 if enable_yum? 504 build_dependencies = [ 505 repositories_dir, 506 rpm_archive_name, 507 yum_build_sh, 508 yum_spec_in_path, 509 ] 510 else 511 build_dependencies = [] 512 end 513 task :build => build_dependencies do 514 yum_build if enable_yum? 515 end 516 517 namespace :build do 518 desc "Open console" 519 task :console => build_dependencies do 520 yum_build(console: true) if enable_yum? 521 end 522 end 523 end 524 525 desc "Release Yum repositories" 526 yum_tasks = [ 527 "yum:build", 528 ] 529 task :yum => yum_tasks 530 end 531 532 def define_version_task 533 namespace :version do 534 desc "Update versions" 535 task :update do 536 update_debian_changelog 537 update_spec 538 end 539 end 540 end 541 542 def package_changelog_message 543 "New upstream release." 544 end 545 546 def packager_name 547 ENV["DEBFULLNAME"] || ENV["NAME"] || guess_packager_name_from_git 548 end 549 550 def guess_packager_name_from_git 551 name = `git config --get user.name`.chomp 552 return name unless name.empty? 553 `git log -n 1 --format=%aN`.chomp 554 end 555 556 def packager_email 557 ENV["DEBEMAIL"] || ENV["EMAIL"] || guess_packager_email_from_git 558 end 559 560 def guess_packager_email_from_git 561 email = `git config --get user.email`.chomp 562 return email unless email.empty? 563 `git log -n 1 --format=%aE`.chomp 564 end 565 566 def update_content(path) 567 if File.exist?(path) 568 content = File.read(path) 569 else 570 content = "" 571 end 572 content = yield(content) 573 File.open(path, "w") do |file| 574 file.puts(content) 575 end 576 end 577 578 def update_debian_changelog 579 return unless enable_apt? 580 581 Dir.glob("debian*") do |debian_dir| 582 update_content("#{debian_dir}/changelog") do |content| 583 <<-CHANGELOG.rstrip 584#{@package} (#{@deb_upstream_version}-#{@deb_release}) unstable; urgency=low 585 586 * New upstream release. 587 588 -- #{packager_name} <#{packager_email}> #{@release_time.rfc2822} 589 590#{content} 591 CHANGELOG 592 end 593 end 594 end 595 596 def update_spec 597 return unless enable_yum? 598 599 release_time = @release_time.strftime("%a %b %d %Y") 600 update_content(yum_spec_in_path) do |content| 601 content = content.sub(/^(%changelog\n)/, <<-CHANGELOG) 602%changelog 603* #{release_time} #{packager_name} <#{packager_email}> - #{@rpm_version}-#{@rpm_release} 604- #{package_changelog_message} 605 606 CHANGELOG 607 content = content.sub(/^(Release:\s+)\d+/, "\\11") 608 content.rstrip 609 end 610 end 611 612 def define_docker_tasks 613 namespace :docker do 614 pull_tasks = [] 615 push_tasks = [] 616 617 (apt_targets + yum_targets).each do |target| 618 distribution, version, architecture = split_target(target) 619 os = "#{distribution}-#{version}" 620 621 namespace :pull do 622 desc "Pull built image for #{target}" 623 task target do 624 docker_pull(os, architecture) 625 end 626 pull_tasks << "docker:pull:#{target}" 627 end 628 629 namespace :push do 630 desc "Push built image for #{target}" 631 task target do 632 docker_push(os, architecture) 633 end 634 push_tasks << "docker:push:#{target}" 635 end 636 end 637 638 desc "Pull built images" 639 task :pull => pull_tasks 640 641 desc "Push built images" 642 task :push => push_tasks 643 end 644 end 645end 646