X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?p=pspp;a=blobdiff_plain;f=build-pspp;h=d6ffcceb18c8a30b943987ed99121c806e13dcef;hp=8180ddd80c1d289ac3785f0b833e4ccf7b902b79;hb=HEAD;hpb=e69ef9cf2879acc8bd4c4b53099af8f75e982ef9 diff --git a/build-pspp b/build-pspp index 8180ddd80c..b449a5c89c 100755 --- a/build-pspp +++ b/build-pspp @@ -1,154 +1,590 @@ -#! /usr/bin/env perl +#! /usr/bin/env python3 -use Getopt::Long qw(:config bundling no_ignore_case); -use POSIX; +import getopt +import os.path +import re +import socket +import subprocess +import sys -use strict; -use warnings; +from datetime import datetime +from pathlib import Path -my $help = 0; -GetOptions ("h|help" => \$help); -usage () if $help; +def backquotes(command): + output = subprocess.check_output(command, shell=True, text=True) + if output.endswith('\n'): + output = output[:-1] + return output -die "$0: exactly one or two nonoption arguments are required\n" - if @ARGV != 1 && @ARGV != 2; -my $buildslave = `hostname`; -chomp $buildslave; +def print_usage(): + print("""\ +%s, for building and testing PSPP +usage: %s [OPTIONS] [TARBALL | REPO REFSPEC] +where TARBALL is the name of a tarball produced by "make dist" + or REPO and REFSPEC are a Git repo and refspec (e.g. branch) to clone. + +Options: + --help Print this usage message and exit + --ssw=TARBALL Get ssw from TARBALL instead of from Git. + --no-binary Build source tarballs but no binaries. + --batch Do not print progress to stdout. + --no-perl Do not build Perl module.""" + % (sys.argv[0], sys.argv[0])) + sys.exit(0) + + +def run(command, id=None): + if not try_run(command, id): + fail() + + +def try_run(command, id=None): + LOG.write("%s\n" % command) + + p = subprocess.Popen(command, shell=True, text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + + start = datetime.now() + est_time = 0 if id is None else read_timing(id) + + lines = 0 + for s in p.stdout: + LOG.write(s) + + elapsed = (datetime.now() - start).seconds + progress = "%d lines logged, %d s elapsed" % (lines, elapsed) + lines += 1 + if est_time > 0: + left = est_time - elapsed + if left > 0: + progress += ", ETA %d s" % left + if not batch: + sys.stdout.write("\r%s%s\r" + % (progress, " " * (79 - len(progress)))) + if not batch: + sys.stdout.write("\r%s\r" % (' ' * 79)) + + if id is not None: + write_timing(id, (datetime.now() - start).seconds) + + rc = p.wait() + if rc == 0: + return True + + if rc < 0: + sys.stderr.write("%s: child died with signal %d\n" % (command, -rc)) + else: + sys.stderr.write("%s: child exited with value %d\n" % (command, rc)) + return False + + +def read_timing(id): + try: + for s in open("%s/timings" % topdir): + m = re.match('([^=]+)=(.*)', s.rstrip()) + if m: + key, value = m.groups() + if key == id: + try: + return int(value) + except ValueError: + return 0 + return 0 + except FileNotFoundError: + return 0 + + +def write_timing(id, time): + tmpname = "%s/timings.tmp%d" % (topdir, os.getpid()) + NEWTIMINGS = open(tmpname, 'w') + try: + OLDTIMINGS = open("%s/timings" % topdir, "r") + for s in OLDTIMINGS: + m = re.match('([^=]+)=(.*)', s.rstrip()) + if m: + key, value = m.groups() + if key == id: + continue + NEWTIMINGS.write(s) + except FileNotFoundError: + pass + NEWTIMINGS.write("%s=%s\n" % (id, time)) + NEWTIMINGS.close() + os.rename(tmpname, "%s/timings" % topdir) + + +def fail(): + set_var("result", "failure") + sys.stderr.write("Build failed, refer to:\n\t%s\nfor details.\n" % logfile) + sys.exit(1) + + +def start_step(msg): + LOG.write(" \n%s\n" % msg) + if not batch: + print(msg) + + +def set_var(var, value): + VARS.write("%s=%s\n" % (var, value)) + if not batch: + print("\t%s=%s" % (var, value)) + + VAR = open("%s/vars/%s" % (resultsdir, var), "wt") + VAR.write("%s\n" % value) + VAR.close() + + +def saved_result(name, product): + start_step("Saving %s: %s" % (name, product)) + + +def save_result(name, src, rm_src=False): + basename = os.path.basename(src) + dst = "%s/%s" % (resultsdir, basename) + + saved_result(name, basename) + run("cp -R %s %s" % (src, dst)) + + if rm_src: + run("rm %s" % src) + + return dst + + +def save_result_if_exists(name, src, rm_src=False): + if Path(src).exists(): + save_result(name, src, rm_src) + else: + start_step("%s does not exist, cannot save" % src) + + +def ref_to_commit(ref, repo='.'): + return backquotes("cd %s && %s rev-parse %s" % (repo, GIT, ref)) + + +def add_commit_to_version(name, commit, dir, extra_news = None): + abbrev_commit = commit[:6] + + # Extract version number. + start_step("Extract %s repository version number" % name) + fields = backquotes("cd %s && autoconf -t AC_INIT" % dir).split(':') + file, line, macro, package, repo_version = fields[:5] + rest = fields[5:] + set_var("%s_repo_version" % name, repo_version) + + # Is this a "gnits" mode tree? + start_step("Checking %s Automake mode" % name) + + am_mode = "gnu" + for s in open("%s/Makefile.am" % dir): + if "gnits" in s: + am_mode = "gnits" + break + LOG.write("%s Automake mode is %s\n" % (name, am_mode)) + + # Generate version number for build. + # We want to append -g012345, but if we're in Gnits mode and the + # version number already has a hyphen, we have to omit it. + start_step("Generate %s build version number" % name) + version = repo_version + if '-' not in version: + version += '-' + version += 'g' + abbrev_commit + set_var("%s_version" % name, version) + + # Append -g012345 to configure.ac version number. + start_step("Updating %s version number in %s" % (name, file)) + fullname = "%s/%s" % (dir, file) + NEWFILE = open("%s.new" % fullname, "w") + ln = 1 + for s in open(fullname): + if ln != int(line): + NEWFILE.write(s) + else: + NEWFILE.write("AC_INIT([%s], [%s]" % (package, version)) + for field in rest: + NEWFILE.write(", [%s]" % field) + NEWFILE.write(")\n") + ln += 1 + NEWFILE.close() + os.rename("%s.new" % fullname, fullname) + + # Add note to beginning of NEWS (otherwise "make dist" fails). + start_step("Updating %s NEWS" % name) + fullname = "%s/NEWS" % name + NEWFILE = open("%s.new" % fullname, "w") + found_changes = False + for s in open(fullname): + if not found_changes and (s.startswith('Changes') or repo_version in s): + found_changes = True + NEWFILE.write("""\ +Changes from %(repo_version)s to %(version)s: + + * Built from PSPP commit %(revision)s + in branch %(branch)s on builder %(builder)s. + +""" + % {'repo_version': repo_version, + 'version': version, + 'revision': commit, + 'branch': branch, + 'builder': builder}) + if extra_news: + NEWFILE.write(extra_news) + NEWFILE.write('\n') + + NEWFILE.write(s) + NEWFILE.close() + os.rename("%s.new" % fullname, fullname) + + return version + +try: + opts, args = getopt.gnu_getopt(sys.argv[1:], "ho:", + ["help", + "binary", "no-binary", + "batch", "no-batch", + "no-perl", + "output=", + "ssw=", + "builder=", "build-number="]) +except getopt.GetoptError as err: + # print help information and exit: + print(err) # will print something like "option -a not recognized" + sys.exit(1) + +build_binary = True +batch = not os.isatty(1) +builddir = None +build_number = None +builder = None +build_perl = True +ssw = "https://git.savannah.gnu.org/git/ssw.git master" +for o, a in opts: + if o in ("-h", "--help"): + print_usage() + elif o == "--binary": + build_binary = True + elif o == "--no-binary": + build_binary = False + elif o == "--batch": + batch = True + elif o == "--no-batch": + batch = False + elif o in ("-o", "--output"): + builddir = a + elif o == "--builder": + builder = a + elif o == "--build-number": + build_number = a + elif o == "--no-perl": + build_perl = False + elif o == "--ssw": + ssw = a + else: + assert False, "unhandled option" +if builder is None: + builder = socket.gethostname() + +if len(args) not in (1, 2): + sys.stderr.write( + "%s: exactly one or two nonoption arguments are required\n" + % sys.argv[0]) + sys.exit(1) + +if len(args) == 1: + tarball = os.path.abspath(args[0]) +else: + pass # Tarball will be generated later. # Select build number. -my $buildnumber = POSIX::strftime("%Y%m%d%H%M%S", localtime); -print "\tBUILDNUMBER=$buildnumber\n"; +if build_number is None: + build_number = datetime.now().strftime("%Y%m%d%H%M%S") + +topdir = os.path.dirname(sys.argv[0]) +if not topdir.startswith("/"): + topdir = "%s/%s" % (os.getcwd(), topdir) # Create build directory. -my $builddir = "builds/$buildnumber"; -print "Create $builddir\n"; -mkdir "builds" or die "builds: mkdir: $!\n" if ! -d "builds"; -mkdir $builddir or die "$builddir: mkdir: $!\n"; +if builddir is None: + builddir = "builds/%s" % build_number + if not Path('builds').is_dir(): + os.mkdir('builds') +builddir = os.path.abspath(builddir) +if not Path(builddir).is_dir(): + os.mkdir(builddir) +os.chdir(builddir) + +resultsdir = "%s/results" % builddir +os.mkdir(resultsdir) +os.mkdir("%s/vars" % resultsdir) + +varsfile = "%s/VARS" % resultsdir +VARS = open(varsfile, "wt", buffering=1) + +logfile = "%s/LOG" % resultsdir +LOG = open(logfile, "wt", buffering=1) + +set_var("builder", builder) +set_var("build_number", build_number) + +GIT = "git --git-dir=%s/.git" % topdir + +if ssw.endswith('.tar.gz'): + ssw_basename = os.path.basename(ssw) + ssw_file = '%s/%s' % (topdir, ssw_basename) + ssw_dir = ssw_basename[:-7] +else: + ssw_dir = 'ssw' +if len(args) == 2: + repo, branch = args + + if ssw.endswith('.tar.gz'): + if not Path(ssw_file).exists(): + start_step("Retrieve spread-sheet-widget tarball %s" % ssw_file) + run("wget -O %s %s" % (ssw_file, ssw)) + elif ' ' in ssw: + ssw_url, ssw_ref = ssw.split() -if (@ARGV == 2) { - my ($repo, $branch) = @ARGV; + start_step("Clone spread-sheet-widget into %s" % ssw_dir) + run("git clone %s %s" % (ssw_url, ssw_dir)) - # Fetch branch - print "Fetch $repo, branch $branch\n"; - run ("git fetch $repo +$branch:buildtmp/$$/pspp"); + start_step("Check out %s in ssw" % ssw_ref) + run("cd %s && git checkout %s" % (ssw_dir, ssw_ref)) + ssw_commit = ref_to_commit("HEAD", "ssw") + set_var("ssw_commit", ssw_commit) + + ssw_version = add_commit_to_version("ssw", ssw_commit, "ssw") + + start_step("Bootstrap ssw") + run("cd ssw && ./bootstrap") + + start_step("Configure ssw source") + run("cd ssw && mkdir _build && cd _build && ../configure", + "configure") + + start_step("Make ssw source tarball") + run("cd ssw/_build && make -j128 dist", "dist") + ssw_dir = "spread-sheet-widget-%s" % ssw_version + ssw_file = "ssw/_build/%s.tar.gz" % ssw_dir + save_result("ssw source distribution", ssw_file) + else: + assert False + + start_step("Extract %s into %s" % (ssw_file, ssw_dir)) + run("tar xzf %s" % ssw_file) + + start_step("Configure spread-sheet-widget") + run("cd %s && ./configure --prefix=''" % ssw_dir) + + start_step("Build spread-sheet-widget") + run("cd %s && make -j$(nproc)" % ssw_dir) + + start_step("Install spread-sheet-widget") + run("cd %s && make -j$(nproc) install DESTDIR=$PWD/inst" % ssw_dir) + + start_step("Fetch branch from Git") + set_var("git_repo", repo) + set_var("git_branch", branch) + run("%s fetch %s +%s:refs/builds/%s/pspp" + % (GIT, repo, branch, build_number)) # Get revision number. - my $revision = `git rev-parse buildtmp/$$/pspp`; - chomp $revision; - print "\tREVISION=$revision\n"; - my $abbrev_revision = substr ($revision, 0, 6); + set_var("pspp_ref", "refs/builds/%s/pspp" % build_number) + revision = ref_to_commit("refs/builds/%s/pspp" % build_number) + set_var("pspp_commit", revision) # Extract source. - print "Extract branch into $builddir/pspp$buildnumber\n"; - run ("git archive --format=tar --prefix=pspp$buildnumber/ buildtmp/$$/pspp | (cd $builddir && tar xf -)"); + start_step("Extract branch into source directory") + run("%s archive --format=tar --prefix=pspp/ refs/builds/%s/pspp | tar xf -" + % (GIT, build_number)) - # Extract version number. - print "Extract version number\n"; - my $trace = `cd $builddir/pspp$buildnumber && autoconf -t AC_INIT`; - chomp $trace; - my ($file, $line, $macro, $package, $version, @rest) = split (':', $trace); - print "\tVERSION=$version\n"; - - # Append -g012345 to AC_INIT version number. - my $fullname = "$builddir/pspp$buildnumber/$file"; - open (OLDFILE, '<', $fullname) - or die "opening $fullname failed: $!\n"; - open (NEWFILE, '>', "$fullname.new") - or die "creating $fullname.new failed: $!\n"; - while () { - if ($. != $line) { - print NEWFILE $_; - } else { - print NEWFILE "AC_INIT([[$package]]"; - print NEWFILE ", [[$version-g$abbrev_revision]]"; - print NEWFILE ", [[$_]]" foreach @rest; - print NEWFILE ")\n"; - } - } - close (NEWFILE); - close (OLDFILE); - rename ("$fullname.new", $fullname) - or die "rename $fullname.new to $fullname failed: $!\n"; + # Get Gnulib commit number. + start_step("Reading README.Git to find Gnulib commit number") + fullname = "pspp/README.Git" + gnulib_commit = None + for s in open(fullname): + m = re.match(r'\s+commit ([0-9a-fA-F]{8,})', s) + if m: + gnulib_commit = m.group(1) + break + if gnulib_commit is None: + sys.stderr.write("%s does not specify a Git commit number\n" + % fullname) + fail() + set_var("gnulib_commit", gnulib_commit) - # Add note to beginning of NEWS (otherwise "make dist" fails). - $fullname = "$builddir/pspp$buildnumber/NEWS"; - open (OLDFILE, '<', $fullname) - or die "opening $fullname failed: $!\n"; - open (NEWFILE, '>', "$fullname.new") - or die "creating $fullname.new failed: $!\n"; - my $found_changes = 0; - while () { - if (!$found_changes && /^Changes/) { - $found_changes = 1; - print NEWFILE <) { - ($gnulib_commit) = /^\s+commit ([0-9a-fA-F]{8,})/ and last; - } - die "$fullname does not specify a Git commit number\n" - if !defined ($gnulib_commit); - print "\tGNULIB_REVISION=$gnulib_commit\n"; -} - -sub usage { - print < /dev/null" + % (GIT, gnulib_commit)) + if rc: + start_step("Updating Gnulib to obtain commit") + run("%s fetch gnulib" % GIT) + run("%s update-ref refs/builds/%s/gnulib %s" + % (GIT, build_number, gnulib_commit)) + set_var("gnulib_ref", "refs/builds/%s/gnulib" % build_number) -Options: - --help Print this usage message and exit -EOF - exit(0); -} - -sub run { - my ($command) = @_; - return if system ($command) == 0; - if ($? == -1) { - print "$command: failed to execute: $!\n"; - } elsif ($? & 127) { - printf "%s: child died with signal %d, %s coredump\n", - $command, ($? & 127), ($? & 128) ? 'with' : 'without'; - } else { - printf "%s: child exited with value %d\n", $command, $? >> 8; - } - exit 1; -} - -# Clone source -# Add build number -# Tag build -# Clone gnulib at correct commit number -# Run gnulib-tool. -# Run configure -# Make dist - -# Unpack dist -# Run configure -# Check -# Install -# Make binary dist -# Build mingw32 installer -# Other distcheck stuff? + # Extract gnulib source. + start_step("Extract Gnulib source") + run("%s archive --format=tar --prefix=gnulib/ %s | tar xf -" + % (GIT, gnulib_commit)) + + # Bootstrap. + start_step("Bootstrap (make -f Smake)") + run("cd pspp && make -f Smake -j$(nproc)", "bootstrap") + + # Configure. + start_step("Configure source") + run("cd pspp && " + "mkdir _build && " + "cd _build && ../configure " + "PKG_CONFIG_PATH=$PWD/../../%s/inst/lib/pkgconfig" % ssw_dir, + "configure") + + # Distribute. + start_step("Make source tarball") + run("cd pspp/_build && make -j128 dist", "dist") + tarname = "pspp-%s.tar.gz" % version + tarball = save_result("source distribution", "pspp/_build/%s" % tarname, 1) + + # Save translation templates. + potfile = "pspp/_build/po/pspp.pot" + if not Path(potfile).exists(): + potfile = "pspp/po/pspp.pot" + save_result("translation templates", potfile) + + # Build examples for user manual. + start_step("Build examples for user manual") + run("cd pspp/_build && make -j$(nproc) figure-spvs figure-txts figure-texis figure-htmls") + + # Build user manual + start_step("Build user manual") + run("cd pspp && " + "GENDOCS_TEMPLATE_DIR=%s %s/gendocs.sh -s doc/pspp.texi -I doc " + "-I _build/doc -o %s/user-manual --email bug-gnu-pspp@gnu.org " + "pspp \"GNU PSPP User Manual\"" % (topdir, topdir, resultsdir), + "user-manual") + saved_result("User Manual", "user-manual") + + # Build developer's guide + start_step("Build developers guide") + run("cd pspp && " + "GENDOCS_TEMPLATE_DIR=%s %s/gendocs.sh -s doc/pspp-dev.texi " + "-I doc -o %s/dev-guide --email bug-gnu-pspp@gnu.org " + "pspp-dev \"GNU PSPP Developers Guide\"" + % (topdir, topdir, resultsdir), "dev-guide") + saved_result("Developers Guide", "dev-guide") +else: + start_step("Starting from %s" % tarball) + +if build_binary: + start_step("Save tarball to Git") + run("GIT_DIR=%s/.git %s/git-import-tar %s refs/builds/%s/dist" + % (topdir, topdir, tarball, build_number), "git-dist") + set_var("dist_ref", "refs/builds/%s/dist" % build_number) + set_var("dist_commit", ref_to_commit("refs/builds/%s/dist" % build_number)) + + start_step("Determining %s target directory" % tarball) + sample_filename = backquotes("zcat %s | tar tf - | head -1" % tarball) + tarball_dir = re.match('(?:[./])*([^/]+)/', sample_filename).group(1) + set_var("dist_dir", tarball_dir) + + start_step("Extracting source tarball") + run("zcat %s | (cd %s && tar xf -)" % (tarball, builddir)) + + start_step("Extracting tar version") + version_line = backquotes("cd %s/%s && ./configure --version | head -1" + % (builddir, tarball_dir)) + version = re.search(r'configure (\S+)$', version_line).group(1) + set_var("dist_version", version) + binary_version = "%s-%s-build%s" % (version, builder, build_number) + set_var("binary_version", binary_version) + + start_step("Configuring") + run("chmod -R a-w %s/%s" % (builddir, tarball_dir)) + run("chmod u+w %s/%s" % (builddir, tarball_dir)) + if build_perl: + run("chmod -R u+w %s/%s/perl-module" % (builddir, tarball_dir)) + run("mkdir %s/%s/_build" % (builddir, tarball_dir)) + run("chmod a-w %s/%s" % (builddir, tarball_dir)) + ok = try_run( + "cd %(builddir)s/%(tarball_dir)s/_build && ../configure " + "--%(perl)s-perl-module --enable-relocatable --prefix='' " + "PKG_CONFIG_PATH=$PWD/../../../source/%(ssw_dir)s/inst/lib/pkgconfig " + "CPPFLAGS=\"-I$PWD/../../../source/%(ssw_dir)s/inst/include\" " + "LDFLAGS=\"-L$PWD/../../../source/%(ssw_dir)s/inst/lib\" " + % {"builddir": builddir, + "tarball_dir": tarball_dir, + "ssw_dir": ssw_dir, + "perl": "with" if build_perl else "without"}, + "bin-configure") + for basename in ("config.h", "config.log"): + save_result_if_exists("build configuration", + "%s/%s/_build/%s" % (builddir, tarball_dir, + basename)) + if not ok: + fail() + + start_step("Build") + run("cd %s/%s/_build && make -j$(nproc)" % (builddir, tarball_dir), "build") + if build_perl: + run("cd %s/%s/_build/perl-module && perl Makefile.PL && make -j$(nproc)" + % (builddir, tarball_dir), "build Perl module") + + start_step("Install") + run("cd %s/%s/_build && make -j$(nproc) install DESTDIR=$PWD/pspp-%s" + % (builddir, tarball_dir, binary_version), "install") + run("cd ../source/%s && make -j$(nproc) install DESTDIR=%s/%s/_build/pspp-%s" + % (ssw_dir, builddir, tarball_dir, binary_version)) + if build_perl: + run("cd %s/%s/_build/perl-module && " + "make -j$(nproc) install DESTDIR=%s/%s/_build/pspp-%s" + % (builddir, tarball_dir, builddir, tarball_dir, binary_version), + "install Perl module") + run("cd %s/%s/_build/perl-module && " + "make -j$(nproc) install DESTDIR=$PWD/inst" + % (builddir, tarball_dir)) + + start_step("Make binary distribution") + run("cd %s/%s/_build && tar cfz pspp-%s.tar.gz pspp-%s" + % (builddir, tarball_dir, binary_version, binary_version)) + save_result("binary distribution", "%s/%s/_build/pspp-%s.tar.gz" + % (builddir, tarball_dir, binary_version), 1) + + start_step("Check") + ok = try_run("cd %s/%s/_build && make check TESTSUITEFLAGS=-j$(nproc)" + % (builddir, tarball_dir), "check") + for basename in ("tests/testsuite.log", "tests/testsuite.dir"): + save_result_if_exists("test logs", "%s/%s/_build/%s" + % (builddir, tarball_dir, basename)) + if not ok: + fail() + + start_step("Uninstall") + run("cd ../source/%s && make -j$(nproc) uninstall DESTDIR=%s/%s/_build/pspp-%s" + % (ssw_dir, builddir, tarball_dir, binary_version)) + run("cd %s/%s/_build && make -j$(nproc) uninstall DESTDIR=$PWD/pspp-%s" + % (builddir, tarball_dir, binary_version), "uninstall") + + start_step("Check uninstall") + if build_perl: + run("(cd %s/%s/_build/perl-module/inst && find -type f -print) | " + "(cd %s/%s/_build/pspp-%s && xargs rm)" + % (builddir, tarball_dir, + builddir, tarball_dir, binary_version), "uninstall Perl module") + run("cd %s/%s/_build && " + "make -j$(nproc) distuninstallcheck distuninstallcheck_dir=$PWD/pspp-%s" + % (builddir, tarball_dir, binary_version), + "distuninstallcheck") + + # distcleancheck + +start_step("Success") +set_var("result", "success")