1#!/usr/bin/env python 2 3# This program makes Debian and RPM repositories for MongoDB, by 4# downloading our tarballs of statically linked executables and 5# insinuating them into Linux packages. It must be run on a 6# Debianoid, since Debian provides tools to make RPMs, but RPM-based 7# systems don't provide debian packaging crud. 8 9# Notes: 10# 11# * Almost anything that you want to be able to influence about how a 12# package construction must be embedded in some file that the 13# packaging tool uses for input (e.g., debian/rules, debian/control, 14# debian/changelog; or the RPM specfile), and the precise details are 15# arbitrary and silly. So this program generates all the relevant 16# inputs to the packaging tools. 17# 18# * Once a .deb or .rpm package is made, there's a separate layer of 19# tools that makes a "repository" for use by the apt/yum layers of 20# package tools. The layouts of these repositories are arbitrary and 21# silly, too. 22# 23# * Before you run the program on a new host, these are the 24# prerequisites: 25# 26# apt-get install dpkg-dev rpm debhelper fakeroot ia32-libs createrepo git-core 27# echo "Now put the dist gnupg signing keys in ~root/.gnupg" 28 29import argparse 30import errno 31from glob import glob 32import os 33import re 34import shutil 35import subprocess 36import sys 37import tempfile 38import time 39 40# The MongoDB names for the architectures we support. 41ARCH_CHOICES=["x86_64", "arm64"] 42 43# Made up names for the flavors of distribution we package for. 44DISTROS=["suse", "debian","redhat","ubuntu", "amazon", "amazon2"] 45 46 47class Spec(object): 48 def __init__(self, ver, gitspec = None, rel = None): 49 self.ver = ver 50 self.gitspec = gitspec 51 self.rel = rel 52 53 # Commit-triggerd version numbers can be in the form: 3.0.7-pre-, or 3.0.7-5-g3b67ac 54 # Patch builds version numbers are in the form: 3.5.5-64-g03945fa-patch-58debcdb3ff1223c9d00005b 55 # 56 def is_nightly(self): 57 return bool(re.search("-$", self.version())) or bool(re.search("\d-\d+-g[0-9a-f]+$", self.version())) 58 59 def is_patch(self): 60 return bool(re.search("\d-\d+-g[0-9a-f]+-patch-[0-9a-f]+$", self.version())) 61 62 def is_rc(self): 63 return bool(re.search("-rc\d+$", self.version())) 64 65 def is_pre_release(self): 66 return self.is_rc() or self.is_nightly() 67 68 def version(self): 69 return self.ver 70 71 def patch_id(self): 72 if self.is_patch(): 73 return re.sub(r'.*-([0-9a-f]+$)', r'\1', self.version()) 74 else: 75 return "none" 76 77 def metadata_gitspec(self): 78 """Git revision to use for spec+control+init+manpage files. 79 The default is the release tag for the version being packaged.""" 80 if(self.gitspec): 81 return self.gitspec 82 else: 83 return 'r' + self.version() 84 85 def version_better_than(self, version_string): 86 # FIXME: this is wrong, but I'm in a hurry. 87 # e.g., "1.8.2" < "1.8.10", "1.8.2" < "1.8.2-rc1" 88 return self.ver > version_string 89 90 def suffix(self): 91 return "-org" if int(self.ver.split(".")[1])%2==0 else "-org-unstable" 92 93 def prelease(self): 94 # NOTE: This is only called for RPM packages, and only after 95 # pversion() below has been called. If you want to change this format 96 # and want DEB packages to match, make sure to update pversion() 97 # below 98 # 99 # "N" is either passed in on the command line, or "1" 100 if self.rel: 101 corenum = self.rel 102 else: 103 corenum = 1 104 105 # Version suffix for RPM packages: 106 # 1) RC's - "0.N.rcX" 107 # 2) Nightly (snapshot) - "0.N.latest" 108 # 3) Patch builds - "0.N.patch.<patch_id>" 109 # 4) Standard release - "N" 110 if self.is_rc(): 111 return "0.%s.%s" % (corenum, re.sub('.*-','',self.version())) 112 elif self.is_nightly(): 113 return "0.%s.latest" % (corenum) 114 elif self.is_patch(): 115 return "0.%s.patch.%s" % (corenum, self.patch_id()) 116 else: 117 return str(corenum) 118 119 def pversion(self, distro): 120 # Note: Debian packages have funny rules about dashes in 121 # version numbers, and RPM simply forbids dashes. pversion 122 # will be the package's version number (but we need to know 123 # our upstream version too). 124 125 # For RPM packages this just returns X.Y.X because of the 126 # aforementioned rules, and prelease (above) adds a suffix later, 127 # so detect this case early 128 if re.search("(suse|redhat|fedora|centos|amazon)", distro.name()): 129 return re.sub("-.*", "", self.version()) 130 131 # For DEB packages, this code sets the full version. If you change 132 # this format and want RPM packages to match make sure you change 133 # prelease above as well 134 if re.search("^(debian|ubuntu)", distro.name()): 135 if self.is_nightly(): 136 ver = re.sub("-.*", "-latest", self.ver) 137 elif self.is_patch(): 138 ver = re.sub("-.*", "", self.ver) + "-patch-" + self.patch_id() 139 else: 140 ver = self.ver 141 142 return re.sub("-", "~", ver) 143 144 raise Exception("BUG: unsupported platform?") 145 146 def branch(self): 147 """Return the major and minor portions of the specified version. 148 For example, if the version is "2.5.5" the branch would be "2.5" 149 """ 150 return ".".join(self.ver.split(".")[0:2]) 151 152class Distro(object): 153 def __init__(self, string): 154 self.n=string 155 156 def name(self): 157 return self.n 158 159 def pkgbase(self): 160 return "mongodb" 161 162 def archname(self, arch): 163 """Return the packaging system's architecture name. 164 Power and x86 have different names for apt/yum (ppc64le/ppc64el 165 and x86_64/amd64) 166 """ 167 if re.search("^(debian|ubuntu)", self.n): 168 if arch == "ppc64le": 169 return "ppc64el" 170 elif arch == "s390x": 171 return "s390x" 172 elif arch == "arm64": 173 return "arm64" 174 elif arch.endswith("86"): 175 return "i386" 176 else: 177 return "amd64" 178 elif re.search("^(suse|centos|redhat|fedora|amazon)", self.n): 179 if arch == "ppc64le": 180 return "ppc64le" 181 elif arch == "s390x": 182 return "s390x" 183 elif arch.endswith("86"): 184 return "i686" 185 else: 186 return "x86_64" 187 else: 188 raise Exception("BUG: unsupported platform?") 189 190 def repodir(self, arch, build_os, spec): 191 """Return the directory where we'll place the package files for 192 (distro, distro_version) in that distro's preferred repository 193 layout (as distinct from where that distro's packaging building 194 tools place the package files). 195 196 Examples: 197 198 repo/apt/ubuntu/dists/precise/mongodb-org/2.5/multiverse/binary-amd64 199 repo/apt/ubuntu/dists/precise/mongodb-org/2.5/multiverse/binary-i386 200 201 repo/apt/ubuntu/dists/trusty/mongodb-org/2.5/multiverse/binary-amd64 202 repo/apt/ubuntu/dists/trusty/mongodb-org/2.5/multiverse/binary-i386 203 204 repo/apt/debian/dists/wheezy/mongodb-org/2.5/main/binary-amd64 205 repo/apt/debian/dists/wheezy/mongodb-org/2.5/main/binary-i386 206 207 repo/yum/redhat/6/mongodb-org/2.5/x86_64 208 yum/redhat/6/mongodb-org/2.5/i386 209 210 repo/zypper/suse/11/mongodb-org/2.5/x86_64 211 zypper/suse/11/mongodb-org/2.5/i386 212 213 """ 214 215 repo_directory = "" 216 217 if spec.is_pre_release(): 218 repo_directory = "testing" 219 else: 220 repo_directory = spec.branch() 221 222 if re.search("^(debian|ubuntu)", self.n): 223 return "repo/apt/%s/dists/%s/mongodb-org/%s/%s/binary-%s/" % (self.n, self.repo_os_version(build_os), repo_directory, self.repo_component(), self.archname(arch)) 224 elif re.search("(redhat|fedora|centos|amazon)", self.n): 225 return "repo/yum/%s/%s/mongodb-org/%s/%s/RPMS/" % (self.n, self.repo_os_version(build_os), repo_directory, self.archname(arch)) 226 elif re.search("(suse)", self.n): 227 return "repo/zypper/%s/%s/mongodb-org/%s/%s/RPMS/" % (self.n, self.repo_os_version(build_os), repo_directory, self.archname(arch)) 228 else: 229 raise Exception("BUG: unsupported platform?") 230 231 def repo_component(self): 232 """Return the name of the section/component/pool we are publishing into - 233 e.g. "multiverse" for Ubuntu, "main" for debian.""" 234 if self.n == 'ubuntu': 235 return "multiverse" 236 elif self.n == 'debian': 237 return "main" 238 else: 239 raise Exception("unsupported distro: %s" % self.n) 240 241 def repo_os_version(self, build_os): 242 """Return an OS version suitable for package repo directory 243 naming - e.g. 5, 6 or 7 for redhat/centos, "precise," "wheezy," etc. 244 for Ubuntu/Debian, 11 for suse, "2013.03" for amazon""" 245 if self.n == 'suse': 246 return re.sub(r'^suse(\d+)$', r'\1', build_os) 247 if self.n == 'redhat': 248 return re.sub(r'^rhel(\d).*$', r'\1', build_os) 249 if self.n == 'amazon': 250 return "2013.03" 251 elif self.n == 'amazon2': 252 return "2017.12" 253 elif self.n == 'ubuntu': 254 if build_os == 'ubuntu1204': 255 return "precise" 256 elif build_os == 'ubuntu1404': 257 return "trusty" 258 elif build_os == 'ubuntu1604': 259 return "xenial" 260 elif build_os == 'ubuntu1804': 261 return "bionic" 262 else: 263 raise Exception("unsupported build_os: %s" % build_os) 264 elif self.n == 'debian': 265 if build_os == 'debian81': 266 return 'jessie' 267 elif build_os == 'debian92': 268 return 'stretch' 269 else: 270 raise Exception("unsupported build_os: %s" % build_os) 271 else: 272 raise Exception("unsupported distro: %s" % self.n) 273 274 def make_pkg(self, build_os, arch, spec, srcdir): 275 if re.search("^(debian|ubuntu)", self.n): 276 return make_deb(self, build_os, arch, spec, srcdir) 277 elif re.search("^(suse|centos|redhat|fedora|amazon)", self.n): 278 return make_rpm(self, build_os, arch, spec, srcdir) 279 else: 280 raise Exception("BUG: unsupported platform?") 281 282 def build_os(self, arch): 283 """Return the build os label in the binary package to download (e.g. "rhel55" for redhat, 284 "ubuntu1204" for ubuntu, "debian81" for debian, "suse11" for suse, etc.)""" 285 # Community builds only support amd64 286 if arch not in ['x86_64', 'ppc64le', 's390x', 'arm64']: 287 raise Exception("BUG: unsupported architecture (%s)" % arch) 288 289 if re.search("(suse)", self.n): 290 return [ "suse11", "suse12" ] 291 elif re.search("(redhat|fedora|centos)", self.n): 292 return [ "rhel80", "rhel70", "rhel71", "rhel72", "rhel62", "rhel55" ] 293 elif self.n in ['amazon', 'amazon2']: 294 return [ self.n ] 295 elif self.n == 'ubuntu': 296 return [ "ubuntu1204", "ubuntu1404", "ubuntu1604", "ubuntu1804"] 297 elif self.n == 'debian': 298 return [ "debian81", "debian92" ] 299 else: 300 raise Exception("BUG: unsupported platform?") 301 302 def release_dist(self, build_os): 303 """Return the release distribution to use in the rpm - "el5" for rhel 5.x, 304 "el6" for rhel 6.x, return anything else unchanged""" 305 306 if self.n == 'amazon': 307 return 'amzn1' 308 elif self.n == 'amazon2': 309 return 'amzn2' 310 else: 311 return re.sub(r'^rh(el\d).*$', r'\1', build_os) 312 313def get_args(distros, arch_choices): 314 315 distro_choices=[] 316 for distro in distros: 317 for arch in arch_choices: 318 distro_choices.extend(distro.build_os(arch)) 319 320 parser = argparse.ArgumentParser(description='Build MongoDB Packages') 321 parser.add_argument("-s", "--server-version", help="Server version to build (e.g. 2.7.8-rc0)", required=True) 322 parser.add_argument("-m", "--metadata-gitspec", help="Gitspec to use for package metadata files", required=False) 323 parser.add_argument("-r", "--release-number", help="RPM release number base", type=int, required=False) 324 parser.add_argument("-d", "--distros", help="Distros to build for", choices=distro_choices, required=False, default=[], action='append') 325 parser.add_argument("-p", "--prefix", help="Directory to build into", required=False) 326 parser.add_argument("-a", "--arches", help="Architecture to build", choices=arch_choices, default=[], required=False, action='append') 327 parser.add_argument("-t", "--tarball", help="Local tarball to package", required=True, type=lambda x: is_valid_file(parser, x)) 328 329 args = parser.parse_args() 330 331 if len(args.distros) * len(args.arches) > 1 and args.tarball: 332 parser.error("Can only specify local tarball with one distro/arch combination") 333 334 return args 335 336def main(argv): 337 338 distros=[Distro(distro) for distro in DISTROS] 339 340 args = get_args(distros, ARCH_CHOICES) 341 342 spec = Spec(args.server_version, args.metadata_gitspec, args.release_number) 343 344 oldcwd=os.getcwd() 345 srcdir=oldcwd+"/../" 346 347 # Where to do all of our work. Use a randomly-created directory if one 348 # is not passed in. 349 prefix = args.prefix 350 if prefix is None: 351 prefix = tempfile.mkdtemp() 352 print "Working in directory %s" % prefix 353 354 os.chdir(prefix) 355 try: 356 # Build a package for each distro/spec/arch tuple, and 357 # accumulate the repository-layout directories. 358 for (distro, arch) in crossproduct(distros, args.arches): 359 360 for build_os in distro.build_os(arch): 361 if build_os in args.distros or not args.distros: 362 363 filename = tarfile(build_os, arch, spec) 364 ensure_dir(filename) 365 shutil.copyfile(args.tarball, filename) 366 367 repo = make_package(distro, build_os, arch, spec, srcdir) 368 make_repo(repo, distro, build_os, spec) 369 370 finally: 371 os.chdir(oldcwd) 372 373def crossproduct(*seqs): 374 """A generator for iterating all the tuples consisting of elements 375 of seqs.""" 376 l = len(seqs) 377 if l == 0: 378 pass 379 elif l == 1: 380 for i in seqs[0]: 381 yield [i] 382 else: 383 for lst in crossproduct(*seqs[:-1]): 384 for i in seqs[-1]: 385 lst2=list(lst) 386 lst2.append(i) 387 yield lst2 388 389def sysassert(argv): 390 """Run argv and assert that it exited with status 0.""" 391 print "In %s, running %s" % (os.getcwd(), " ".join(argv)) 392 sys.stdout.flush() 393 sys.stderr.flush() 394 assert(subprocess.Popen(argv).wait()==0) 395 396def backtick(argv): 397 """Run argv and return its output string.""" 398 print "In %s, running %s" % (os.getcwd(), " ".join(argv)) 399 sys.stdout.flush() 400 sys.stderr.flush() 401 return subprocess.Popen(argv, stdout=subprocess.PIPE).communicate()[0] 402 403def tarfile(build_os, arch, spec): 404 """Return the location where we store the downloaded tarball for 405 this package""" 406 return "dl/mongodb-linux-%s-%s-%s.tar.gz" % (spec.version(), build_os, arch) 407 408def setupdir(distro, build_os, arch, spec): 409 # The setupdir will be a directory containing all inputs to the 410 # distro's packaging tools (e.g., package metadata files, init 411 # scripts, etc), along with the already-built binaries). In case 412 # the following format string is unclear, an example setupdir 413 # would be dst/x86_64/debian-sysvinit/wheezy/mongodb-org-unstable/ 414 # or dst/x86_64/redhat/rhel55/mongodb-org-unstable/ 415 return "dst/%s/%s/%s/%s%s-%s/" % (arch, distro.name(), build_os, distro.pkgbase(), spec.suffix(), spec.pversion(distro)) 416 417def unpack_binaries_into(build_os, arch, spec, where): 418 """Unpack the tarfile for (build_os, arch, spec) into directory where.""" 419 rootdir=os.getcwd() 420 ensure_dir(where) 421 # Note: POSIX tar doesn't require support for gtar's "-C" option, 422 # and Python's tarfile module prior to Python 2.7 doesn't have the 423 # features to make this detail easy. So we'll just do the dumb 424 # thing and chdir into where and run tar there. 425 os.chdir(where) 426 try: 427 sysassert(["tar", "xvzf", rootdir+"/"+tarfile(build_os, arch, spec)]) 428 release_dir = glob('mongodb-linux-*')[0] 429 for releasefile in "bin", "LICENSE-Community.txt", "README", "THIRD-PARTY-NOTICES", "THIRD-PARTY-NOTICES.gotools", "MPL-2": 430 print "moving file: %s/%s" % (release_dir, releasefile) 431 os.rename("%s/%s" % (release_dir, releasefile), releasefile) 432 os.rmdir(release_dir) 433 except Exception: 434 exc=sys.exc_value 435 os.chdir(rootdir) 436 raise exc 437 os.chdir(rootdir) 438 439def make_package(distro, build_os, arch, spec, srcdir): 440 """Construct the package for (arch, distro, spec), getting 441 packaging files from srcdir and any user-specified suffix from 442 suffixes""" 443 444 sdir=setupdir(distro, build_os, arch, spec) 445 ensure_dir(sdir) 446 # Note that the RPM packages get their man pages from the debian 447 # directory, so the debian directory is needed in all cases (and 448 # innocuous in the debianoids' sdirs). 449 for pkgdir in ["debian", "rpm"]: 450 print "Copying packaging files from %s to %s" % ("%s/%s" % (srcdir, pkgdir), sdir) 451 # FIXME: sh-dash-cee is bad. See if tarfile can do this. 452 sysassert(["sh", "-c", "(cd \"%s\" && tar cf - %s/ ) | (cd \"%s\" && tar xvf -)" % (srcdir, pkgdir, sdir)]) 453 # Splat the binaries under sdir. The "build" stages of the 454 # packaging infrastructure will move the files to wherever they 455 # need to go. 456 unpack_binaries_into(build_os, arch, spec, sdir) 457 # Remove the mongoreplay binary due to libpcap dynamic 458 # linkage. 459 if os.path.exists(sdir + "bin/mongoreplay"): 460 os.unlink(sdir + "bin/mongoreplay") 461 return distro.make_pkg(build_os, arch, spec, srcdir) 462 463def make_repo(repodir, distro, build_os, spec): 464 if re.search("(debian|ubuntu)", repodir): 465 make_deb_repo(repodir, distro, build_os, spec) 466 elif re.search("(suse|centos|redhat|fedora|amazon)", repodir): 467 make_rpm_repo(repodir) 468 else: 469 raise Exception("BUG: unsupported platform?") 470 471def make_deb(distro, build_os, arch, spec, srcdir): 472 # I can't remember the details anymore, but the initscript/upstart 473 # job files' names must match the package name in some way; and 474 # see also the --name flag to dh_installinit in the generated 475 # debian/rules file. 476 suffix=spec.suffix() 477 sdir=setupdir(distro, build_os, arch, spec) 478 if re.search("debian", distro.name()): 479 os.unlink(sdir+"debian/mongod.upstart") 480 os.link(sdir+"debian/mongod.service", sdir+"debian/%s%s-server.mongod.service" % (distro.pkgbase(), suffix)) 481 os.unlink(sdir+"debian/init.d") 482 elif re.search("ubuntu", distro.name()): 483 os.unlink(sdir+"debian/init.d") 484 if build_os in ("ubuntu1204", "ubuntu1404", "ubuntu1410"): 485 os.link(sdir+"debian/mongod.upstart", sdir+"debian/%s%s-server.mongod.upstart" % (distro.pkgbase(), suffix)) 486 os.unlink(sdir+"debian/mongod.service") 487 else: 488 os.link(sdir+"debian/mongod.service", sdir+"debian/%s%s-server.mongod.service" % (distro.pkgbase(), suffix)) 489 os.unlink(sdir+"debian/mongod.upstart") 490 else: 491 raise Exception("unknown debianoid flavor: not debian or ubuntu?") 492 # Rewrite the control and rules files 493 write_debian_changelog(sdir+"debian/changelog", spec, srcdir) 494 distro_arch=distro.archname(arch) 495 sysassert(["cp", "-v", srcdir+"debian/%s%s.control" % (distro.pkgbase(), suffix), sdir+"debian/control"]) 496 sysassert(["cp", "-v", srcdir+"debian/%s%s.rules" % (distro.pkgbase(), suffix), sdir+"debian/rules"]) 497 498 499 # old non-server-package postinst will be hanging around for old versions 500 # 501 if os.path.exists(sdir+"debian/postinst"): 502 os.unlink(sdir+"debian/postinst") 503 504 # copy our postinst files 505 # 506 sysassert(["sh", "-c", "cp -v \"%sdebian/\"*.postinst \"%sdebian/\""%(srcdir, sdir)]) 507 508 # Do the packaging. 509 oldcwd=os.getcwd() 510 try: 511 os.chdir(sdir) 512 sysassert(["dpkg-buildpackage", "-uc", "-us", "-a" + distro_arch]) 513 finally: 514 os.chdir(oldcwd) 515 r=distro.repodir(arch, build_os, spec) 516 ensure_dir(r) 517 # FIXME: see if shutil.copyfile or something can do this without 518 # much pain. 519 sysassert(["sh", "-c", "cp -v \"%s/../\"*.deb \"%s\""%(sdir, r)]) 520 return r 521 522def make_deb_repo(repo, distro, build_os, spec): 523 # Note: the Debian repository Packages files must be generated 524 # very carefully in order to be usable. 525 oldpwd=os.getcwd() 526 os.chdir(repo+"../../../../../../") 527 try: 528 dirs=set([os.path.dirname(deb)[2:] for deb in backtick(["find", ".", "-name", "*.deb"]).split()]) 529 for d in dirs: 530 s=backtick(["dpkg-scanpackages", d, "/dev/null"]) 531 with open(d+"/Packages", "w") as f: 532 f.write(s) 533 b=backtick(["gzip", "-9c", d+"/Packages"]) 534 with open(d+"/Packages.gz", "wb") as f: 535 f.write(b) 536 finally: 537 os.chdir(oldpwd) 538 # Notes: the Release{,.gpg} files must live in a special place, 539 # and must be created after all the Packages.gz files have been 540 # done. 541 s="""Origin: mongodb 542Label: mongodb 543Suite: %s 544Codename: %s/mongodb-org 545Architectures: amd64 arm64 546Components: %s 547Description: MongoDB packages 548""" % (distro.repo_os_version(build_os), distro.repo_os_version(build_os), distro.repo_component()) 549 if os.path.exists(repo+"../../Release"): 550 os.unlink(repo+"../../Release") 551 if os.path.exists(repo+"../../Release.gpg"): 552 os.unlink(repo+"../../Release.gpg") 553 oldpwd=os.getcwd() 554 os.chdir(repo+"../../") 555 s2=backtick(["apt-ftparchive", "release", "."]) 556 try: 557 with open("Release", 'w') as f: 558 f.write(s) 559 f.write(s2) 560 finally: 561 os.chdir(oldpwd) 562 563 564def move_repos_into_place(src, dst): 565 # Find all the stuff in src/*, move it to a freshly-created 566 # directory beside dst, then play some games with symlinks so that 567 # dst is a name the new stuff and dst+".old" names the previous 568 # one. This feels like a lot of hooey for something so trivial. 569 570 # First, make a crispy fresh new directory to put the stuff in. 571 i=0 572 while True: 573 date_suffix=time.strftime("%Y-%m-%d") 574 dname=dst+".%s.%d" % (date_suffix, i) 575 try: 576 os.mkdir(dname) 577 break 578 except OSError: 579 exc=sys.exc_value 580 if exc.errno == errno.EEXIST: 581 pass 582 else: 583 raise exc 584 i=i+1 585 586 # Put the stuff in our new directory. 587 for r in os.listdir(src): 588 sysassert(["cp", "-rv", src + "/" + r, dname]) 589 590 # Make a symlink to the new directory; the symlink will be renamed 591 # to dst shortly. 592 i=0 593 while True: 594 tmpnam=dst+".TMP.%d" % i 595 try: 596 os.symlink(dname, tmpnam) 597 break 598 except OSError: # as exc: # Python >2.5 599 exc=sys.exc_value 600 if exc.errno == errno.EEXIST: 601 pass 602 else: 603 raise exc 604 i=i+1 605 606 # Make a symlink to the old directory; this symlink will be 607 # renamed shortly, too. 608 oldnam=None 609 if os.path.exists(dst): 610 i=0 611 while True: 612 oldnam=dst+".old.%d" % i 613 try: 614 os.symlink(os.readlink(dst), oldnam) 615 break 616 except OSError: # as exc: # Python >2.5 617 exc=sys.exc_value 618 if exc.errno == errno.EEXIST: 619 pass 620 else: 621 raise exc 622 623 os.rename(tmpnam, dst) 624 if oldnam: 625 os.rename(oldnam, dst+".old") 626 627 628def write_debian_changelog(path, spec, srcdir): 629 oldcwd=os.getcwd() 630 os.chdir(srcdir) 631 preamble="" 632 try: 633 s=preamble+backtick(["sh", "-c", "git archive %s debian/changelog | tar xOf -" % spec.metadata_gitspec()]) 634 finally: 635 os.chdir(oldcwd) 636 lines=s.split("\n") 637 # If the first line starts with "mongodb", it's not a revision 638 # preamble, and so frob the version number. 639 lines[0]=re.sub("^mongodb \\(.*\\)", "mongodb (%s)" % (spec.pversion(Distro("debian"))), lines[0]) 640 # Rewrite every changelog entry starting in mongodb<space> 641 lines=[re.sub("^mongodb ", "mongodb%s " % (spec.suffix()), l) for l in lines] 642 lines=[re.sub("^ --", " --", l) for l in lines] 643 s="\n".join(lines) 644 with open(path, 'w') as f: 645 f.write(s) 646 647def make_rpm(distro, build_os, arch, spec, srcdir): 648 # Create the specfile. 649 suffix=spec.suffix() 650 sdir=setupdir(distro, build_os, arch, spec) 651 652 specfile = srcdir + "rpm/mongodb%s.spec" % suffix 653 init_spec = specfile.replace(".spec", "-init.spec") 654 655 # The Debian directory is here for the manpages so we we need to remove the service file 656 # from it so that RPM packages don't end up with the Debian file. 657 os.unlink(sdir + "debian/mongod.service") 658 659 # Swap out systemd files, different systemd spec files, and init scripts as needed based on 660 # underlying os version. Arranged so that new distros moving forward automatically use 661 # systemd. Note: the SUSE init packages use a different init script than then other RPM 662 # distros. 663 # 664 if distro.name() == "suse" and distro.repo_os_version(build_os) in ("10", "11"): 665 os.unlink(sdir+"rpm/init.d-mongod") 666 os.link(sdir+"rpm/init.d-mongod.suse", sdir+"rpm/init.d-mongod") 667 668 os.unlink(specfile) 669 os.link(init_spec, specfile) 670 elif distro.name() == "redhat" and distro.repo_os_version(build_os) in ("5", "6"): 671 os.unlink(specfile) 672 os.link(init_spec, specfile) 673 elif distro.name() == "amazon": 674 os.unlink(specfile) 675 os.link(init_spec, specfile) 676 677 topdir=ensure_dir('%s/rpmbuild/%s/' % (os.getcwd(), build_os)) 678 for subdir in ["BUILD", "RPMS", "SOURCES", "SPECS", "SRPMS"]: 679 ensure_dir("%s/%s/" % (topdir, subdir)) 680 distro_arch=distro.archname(arch) 681 # RPM tools take these macro files that define variables in 682 # RPMland. Unfortunately, there's no way to tell RPM tools to use 683 # a given file *in addition* to the files that it would already 684 # load, so we have to figure out what it would normally load, 685 # augment that list, and tell RPM to use the augmented list. To 686 # figure out what macrofiles ordinarily get loaded, older RPM 687 # versions had a parameter called "macrofiles" that could be 688 # extracted from "rpm --showrc". But newer RPM versions don't 689 # have this. To tell RPM what macros to use, older versions of 690 # RPM have a --macros option that doesn't work; on these versions, 691 # you can put a "macrofiles" parameter into an rpmrc file. But 692 # that "macrofiles" setting doesn't do anything for newer RPM 693 # versions, where you have to use the --macros flag instead. And 694 # all of this is to let us do our work with some guarantee that 695 # we're not clobbering anything that doesn't belong to us. 696 # 697 # On RHEL systems, --rcfile will generally be used and 698 # --macros will be used in Ubuntu. 699 # 700 macrofiles=[l for l in backtick(["rpm", "--showrc"]).split("\n") if l.startswith("macrofiles")] 701 flags=[] 702 macropath=os.getcwd()+"/macros" 703 704 write_rpm_macros_file(macropath, topdir, distro.release_dist(build_os)) 705 if len(macrofiles)>0: 706 macrofiles=macrofiles[0]+":"+macropath 707 rcfile=os.getcwd()+"/rpmrc" 708 write_rpmrc_file(rcfile, macrofiles) 709 flags=["--rcfile", rcfile] 710 else: 711 # This hard-coded hooey came from some box running RPM 712 # 4.4.2.3. It may not work over time, but RPM isn't sanely 713 # configurable. 714 flags=["--macros", "/usr/lib/rpm/macros:/usr/lib/rpm/%s-linux/macros:/usr/lib/rpm/suse/macros:/etc/rpm/macros.*:/etc/rpm/macros:/etc/rpm/%s-linux/macros:~/.rpmmacros:%s" % (distro_arch, distro_arch, macropath)] 715 # Put the specfile and the tar'd up binaries and stuff in 716 # place. 717 # 718 # The version of rpm and rpm tools in RHEL 5.5 can't interpolate the 719 # %{dynamic_version} macro, so do it manually 720 with open(specfile, "r") as spec_source: 721 with open(topdir+"SPECS/" + os.path.basename(specfile), "w") as spec_dest: 722 for line in spec_source: 723 line = line.replace('%{dynamic_version}', spec.pversion(distro)) 724 line = line.replace('%{dynamic_release}', spec.prelease()) 725 spec_dest.write(line) 726 727 oldcwd=os.getcwd() 728 os.chdir(sdir+"/../") 729 try: 730 sysassert(["tar", "-cpzf", topdir+"SOURCES/mongodb%s-%s.tar.gz" % (suffix, spec.pversion(distro)), os.path.basename(os.path.dirname(sdir))]) 731 finally: 732 os.chdir(oldcwd) 733 # Do the build. 734 735 flags.extend(["-D", "dynamic_version " + spec.pversion(distro), "-D", "dynamic_release " + spec.prelease(), "-D", "_topdir " + topdir]) 736 737 # Versions of RPM after 4.4 ignore our BuildRoot tag so we need to 738 # specify it on the command line args to rpmbuild 739 if ((distro.name() == "suse" and distro.repo_os_version(build_os) == "15") 740 or (distro.name() == "redhat" and distro.repo_os_version(build_os) == "8")): 741 flags.extend([ 742 "--buildroot", os.path.join(topdir, "BUILDROOT"), 743 ]) 744 745 sysassert(["rpmbuild", "-ba", "--target", distro_arch] + flags + 746 ["%s/SPECS/mongodb%s.spec" % (topdir, suffix)]) 747 repo_dir = distro.repodir(arch, build_os, spec) 748 ensure_dir(repo_dir) 749 750 # FIXME: see if some combination of shutil.copy<hoohah> and glob 751 # can do this without shelling out. 752 sysassert(["sh", "-c", "cp -v \"%s/RPMS/%s/\"*.rpm \"%s\""%(topdir, distro_arch, repo_dir)]) 753 return repo_dir 754 755def make_rpm_repo(repo): 756 oldpwd=os.getcwd() 757 os.chdir(repo+"../") 758 try: 759 sysassert(["createrepo", "."]) 760 finally: 761 os.chdir(oldpwd) 762 763 764def write_rpmrc_file(path, string): 765 with open(path, 'w') as f: 766 f.write(string) 767 768def write_rpm_macros_file(path, topdir, release_dist): 769 with open(path, 'w') as f: 770 f.write("%%_topdir %s\n" % topdir) 771 f.write("%%dist .%s\n" % release_dist) 772 f.write("%_use_internal_dependency_generator 0\n") 773 774def ensure_dir(filename): 775 """Make sure that the directory that's the dirname part of 776 filename exists, and return filename.""" 777 dirpart = os.path.dirname(filename) 778 try: 779 os.makedirs(dirpart) 780 except OSError: # as exc: # Python >2.5 781 exc=sys.exc_value 782 if exc.errno == errno.EEXIST: 783 pass 784 else: 785 raise exc 786 return filename 787 788def is_valid_file(parser, filename): 789 """Check if file exists, and return the filename""" 790 if not os.path.exists(filename): 791 parser.error("The file %s does not exist!" % filename) 792 else: 793 return filename 794 795if __name__ == "__main__": 796 main(sys.argv) 797