1 #! /usr/bin/env python3
10 from datetime import datetime
11 from pathlib import Path
14 def backquotes(command):
15 output = subprocess.check_output(command, shell=True, text=True)
16 if output.endswith('\n'):
23 %s, for building and testing PSPP
24 usage: %s [OPTIONS] [TARBALL | REPO REFSPEC]
25 where TARBALL is the name of a tarball produced by "make dist"
26 or REPO and REFSPEC are a Git repo and refspec (e.g. branch) to clone.
29 --help Print this usage message and exit
30 --ssw=TARBALL Get ssw from TARBALL instead of from Git.
31 --no-binary Build source tarballs but no binaries.
32 --batch Do not print progress to stdout.
33 --no-perl Do not build Perl module."""
34 % (sys.argv[0], sys.argv[0]))
38 def run(command, id=None):
39 if not try_run(command, id):
43 def try_run(command, id=None):
44 LOG.write("%s\n" % command)
46 p = subprocess.Popen(command, shell=True, text=True,
47 stdout=subprocess.PIPE,
48 stderr=subprocess.STDOUT)
50 start = datetime.now()
51 est_time = 0 if id is None else read_timing(id)
57 elapsed = (datetime.now() - start).seconds
58 progress = "%d lines logged, %d s elapsed" % (lines, elapsed)
61 left = est_time - elapsed
63 progress += ", ETA %d s" % left
65 sys.stdout.write("\r%s%s\r"
66 % (progress, " " * (79 - len(progress))))
68 sys.stdout.write("\r%s\r" % (' ' * 79))
71 write_timing(id, (datetime.now() - start).seconds)
78 sys.stderr.write("%s: child died with signal %d\n" % (command, -rc))
80 sys.stderr.write("%s: child exited with value %d\n" % (command, rc))
86 for s in open("%s/timings" % topdir):
87 m = re.match('([^=]+)=(.*)', s.rstrip())
89 key, value = m.groups()
96 except FileNotFoundError:
100 def write_timing(id, time):
101 tmpname = "%s/timings.tmp%d" % (topdir, os.getpid())
102 NEWTIMINGS = open(tmpname, 'w')
104 OLDTIMINGS = open("%s/timings" % topdir, "r")
106 m = re.match('([^=]+)=(.*)', s.rstrip())
108 key, value = m.groups()
112 except FileNotFoundError:
114 NEWTIMINGS.write("%s=%s\n" % (id, time))
116 os.rename(tmpname, "%s/timings" % topdir)
120 set_var("result", "failure")
121 sys.stderr.write("Build failed, refer to:\n\t%s\nfor details.\n" % logfile)
126 LOG.write("
\f\n%s\n" % msg)
131 def set_var(var, value):
132 VARS.write("%s=%s\n" % (var, value))
134 print("\t%s=%s" % (var, value))
136 VAR = open("%s/vars/%s" % (resultsdir, var), "wt")
137 VAR.write("%s\n" % value)
141 def saved_result(name, product):
142 start_step("Saving %s: %s" % (name, product))
145 def save_result(name, src, rm_src=False):
146 basename = os.path.basename(src)
147 dst = "%s/%s" % (resultsdir, basename)
149 saved_result(name, basename)
150 run("cp -R %s %s" % (src, dst))
158 def save_result_if_exists(name, src, rm_src=False):
159 if Path(src).exists():
160 save_result(name, src, rm_src)
162 start_step("%s does not exist, cannot save" % src)
165 def ref_to_commit(ref, repo='.'):
166 return backquotes("cd %s && %s rev-parse %s" % (repo, GIT, ref))
169 def add_commit_to_version(name, commit, dir, extra_news = None):
170 abbrev_commit = commit[:6]
172 # Extract version number.
173 start_step("Extract %s repository version number" % name)
174 fields = backquotes("cd %s && autoconf -t AC_INIT" % dir).split(':')
175 file, line, macro, package, repo_version = fields[:5]
177 set_var("%s_repo_version" % name, repo_version)
179 # Is this a "gnits" mode tree?
180 start_step("Checking %s Automake mode" % name)
183 for s in open("%s/Makefile.am" % dir):
187 LOG.write("%s Automake mode is %s\n" % (name, am_mode))
189 # Generate version number for build.
190 # We want to append -g012345, but if we're in Gnits mode and the
191 # version number already has a hyphen, we have to omit it.
192 start_step("Generate %s build version number" % name)
193 version = repo_version
194 if '-' not in version:
196 version += 'g' + abbrev_commit
197 set_var("%s_version" % name, version)
199 # Append -g012345 to configure.ac version number.
200 start_step("Updating %s version number in %s" % (name, file))
201 fullname = "%s/%s" % (dir, file)
202 NEWFILE = open("%s.new" % fullname, "w")
204 for s in open(fullname):
208 NEWFILE.write("AC_INIT([%s], [%s]" % (package, version))
210 NEWFILE.write(", [%s]" % field)
214 os.rename("%s.new" % fullname, fullname)
216 # Add note to beginning of NEWS (otherwise "make dist" fails).
217 start_step("Updating %s NEWS" % name)
218 fullname = "%s/NEWS" % name
219 NEWFILE = open("%s.new" % fullname, "w")
220 found_changes = False
221 for s in open(fullname):
222 if not found_changes and (s.startswith('Changes') or repo_version in s):
225 Changes from %(repo_version)s to %(version)s:
227 * Built from PSPP commit %(revision)s
228 in branch %(branch)s on builder %(builder)s.
231 % {'repo_version': repo_version,
237 NEWFILE.write(extra_news)
242 os.rename("%s.new" % fullname, fullname)
247 opts, args = getopt.gnu_getopt(sys.argv[1:], "ho:",
249 "binary", "no-binary",
254 "builder=", "build-number="])
255 except getopt.GetoptError as err:
256 # print help information and exit:
257 print(err) # will print something like "option -a not recognized"
261 batch = not os.isatty(1)
266 ssw = "https://git.savannah.gnu.org/git/ssw.git master"
268 if o in ("-h", "--help"):
270 elif o == "--binary":
272 elif o == "--no-binary":
276 elif o == "--no-batch":
278 elif o in ("-o", "--output"):
280 elif o == "--builder":
282 elif o == "--build-number":
284 elif o == "--no-perl":
289 assert False, "unhandled option"
291 builder = socket.gethostname()
293 if len(args) not in (1, 2):
295 "%s: exactly one or two nonoption arguments are required\n"
300 tarball = os.path.abspath(args[0])
302 pass # Tarball will be generated later.
304 # Select build number.
305 if build_number is None:
306 build_number = datetime.now().strftime("%Y%m%d%H%M%S")
308 topdir = os.path.dirname(sys.argv[0])
309 if not topdir.startswith("/"):
310 topdir = "%s/%s" % (os.getcwd(), topdir)
312 # Create build directory.
314 builddir = "builds/%s" % build_number
315 if not Path('builds').is_dir():
317 builddir = os.path.abspath(builddir)
318 if not Path(builddir).is_dir():
322 resultsdir = "%s/results" % builddir
324 os.mkdir("%s/vars" % resultsdir)
326 varsfile = "%s/VARS" % resultsdir
327 VARS = open(varsfile, "wt", buffering=1)
329 logfile = "%s/LOG" % resultsdir
330 LOG = open(logfile, "wt", buffering=1)
332 set_var("builder", builder)
333 set_var("build_number", build_number)
335 GIT = "git --git-dir=%s/.git" % topdir
337 if ssw.endswith('.tar.gz'):
338 ssw_basename = os.path.basename(ssw)
339 ssw_file = '%s/%s' % (topdir, ssw_basename)
340 ssw_dir = ssw_basename[:-7]
346 if ssw.endswith('.tar.gz'):
347 if not Path(ssw_file).exists():
348 start_step("Retrieve spread-sheet-widget tarball %s" % ssw_file)
349 run("wget -O %s %s" % (ssw_file, ssw))
351 ssw_url, ssw_ref = ssw.split()
353 start_step("Clone spread-sheet-widget into %s" % ssw_dir)
354 run("git clone %s %s" % (ssw_url, ssw_dir))
356 start_step("Check out %s in ssw" % ssw_ref)
357 run("cd %s && git checkout %s" % (ssw_dir, ssw_ref))
358 ssw_commit = ref_to_commit("HEAD", "ssw")
359 set_var("ssw_commit", ssw_commit)
361 ssw_version = add_commit_to_version("ssw", ssw_commit, "ssw")
363 start_step("Bootstrap ssw")
364 run("cd ssw && ./bootstrap")
366 start_step("Configure ssw source")
367 run("cd ssw && mkdir _build && cd _build && ../configure",
370 start_step("Make ssw source tarball")
371 run("cd ssw/_build && make -j128 dist", "dist")
372 ssw_dir = "spread-sheet-widget-%s" % ssw_version
373 ssw_file = "ssw/_build/%s.tar.gz" % ssw_dir
374 save_result("ssw source distribution", ssw_file)
378 start_step("Extract %s into %s" % (ssw_file, ssw_dir))
379 run("tar xzf %s" % ssw_file)
381 start_step("Configure spread-sheet-widget")
382 run("cd %s && ./configure --prefix=''" % ssw_dir)
384 start_step("Build spread-sheet-widget")
385 run("cd %s && make -j$(nproc)" % ssw_dir)
387 start_step("Install spread-sheet-widget")
388 run("cd %s && make -j$(nproc) install DESTDIR=$PWD/inst" % ssw_dir)
390 start_step("Fetch branch from Git")
391 set_var("git_repo", repo)
392 set_var("git_branch", branch)
393 run("%s fetch %s +%s:refs/builds/%s/pspp"
394 % (GIT, repo, branch, build_number))
396 # Get revision number.
397 set_var("pspp_ref", "refs/builds/%s/pspp" % build_number)
398 revision = ref_to_commit("refs/builds/%s/pspp" % build_number)
399 set_var("pspp_commit", revision)
402 start_step("Extract branch into source directory")
403 run("%s archive --format=tar --prefix=pspp/ refs/builds/%s/pspp | tar xf -"
404 % (GIT, build_number))
406 # Get Gnulib commit number.
407 start_step("Reading README.Git to find Gnulib commit number")
408 fullname = "pspp/README.Git"
410 for s in open(fullname):
411 m = re.match(r'\s+commit ([0-9a-fA-F]{8,})', s)
413 gnulib_commit = m.group(1)
415 if gnulib_commit is None:
416 sys.stderr.write("%s does not specify a Git commit number\n"
419 set_var("gnulib_commit", gnulib_commit)
421 version = add_commit_to_version("pspp", revision, "pspp",
422 " * Built from Gnulib commit %(gnulib_commit)s.\n")
424 # If we don't already have that Gnulib commit, update Gnulib.
425 rc = os.system("%s rev-parse --verify --quiet %s^0 > /dev/null"
426 % (GIT, gnulib_commit))
428 start_step("Updating Gnulib to obtain commit")
429 run("%s fetch gnulib" % GIT)
430 run("%s update-ref refs/builds/%s/gnulib %s"
431 % (GIT, build_number, gnulib_commit))
432 set_var("gnulib_ref", "refs/builds/%s/gnulib" % build_number)
434 # Extract gnulib source.
435 start_step("Extract Gnulib source")
436 run("%s archive --format=tar --prefix=gnulib/ %s | tar xf -"
437 % (GIT, gnulib_commit))
440 start_step("Bootstrap (make -f Smake)")
441 run("cd pspp && make -f Smake -j$(nproc)", "bootstrap")
444 start_step("Configure source")
447 "cd _build && ../configure "
448 "PKG_CONFIG_PATH=$PWD/../../%s/inst/lib/pkgconfig" % ssw_dir,
452 start_step("Make source tarball")
453 run("cd pspp/_build && make -j128 dist", "dist")
454 tarname = "pspp-%s.tar.gz" % version
455 tarball = save_result("source distribution", "pspp/_build/%s" % tarname, 1)
457 # Save translation templates.
458 potfile = "pspp/_build/po/pspp.pot"
459 if not Path(potfile).exists():
460 potfile = "pspp/po/pspp.pot"
461 save_result("translation templates", potfile)
463 # Build examples for user manual.
464 start_step("Build examples for user manual")
465 run("cd pspp/_build && make -j$(nproc) figure-spvs figure-txts figure-texis figure-htmls")
468 start_step("Build user manual")
470 "GENDOCS_TEMPLATE_DIR=%s %s/gendocs.sh -s doc/pspp.texi -I doc "
471 "-I _build/doc -o %s/user-manual --email bug-gnu-pspp@gnu.org "
472 "pspp \"GNU PSPP User Manual\"" % (topdir, topdir, resultsdir),
474 saved_result("User Manual", "user-manual")
476 # Build developer's guide
477 start_step("Build developers guide")
479 "GENDOCS_TEMPLATE_DIR=%s %s/gendocs.sh -s doc/pspp-dev.texi "
480 "-I doc -o %s/dev-guide --email bug-gnu-pspp@gnu.org "
481 "pspp-dev \"GNU PSPP Developers Guide\""
482 % (topdir, topdir, resultsdir), "dev-guide")
483 saved_result("Developers Guide", "dev-guide")
485 start_step("Starting from %s" % tarball)
488 start_step("Save tarball to Git")
489 run("GIT_DIR=%s/.git %s/git-import-tar %s refs/builds/%s/dist"
490 % (topdir, topdir, tarball, build_number), "git-dist")
491 set_var("dist_ref", "refs/builds/%s/dist" % build_number)
492 set_var("dist_commit", ref_to_commit("refs/builds/%s/dist" % build_number))
494 start_step("Determining %s target directory" % tarball)
495 sample_filename = backquotes("zcat %s | tar tf - | head -1" % tarball)
496 tarball_dir = re.match('(?:[./])*([^/]+)/', sample_filename).group(1)
497 set_var("dist_dir", tarball_dir)
499 start_step("Extracting source tarball")
500 run("zcat %s | (cd %s && tar xf -)" % (tarball, builddir))
502 start_step("Extracting tar version")
503 version_line = backquotes("cd %s/%s && ./configure --version | head -1"
504 % (builddir, tarball_dir))
505 version = re.search(r'configure (\S+)$', version_line).group(1)
506 set_var("dist_version", version)
507 binary_version = "%s-%s-build%s" % (version, builder, build_number)
508 set_var("binary_version", binary_version)
510 start_step("Configuring")
511 run("chmod -R a-w %s/%s" % (builddir, tarball_dir))
512 run("chmod u+w %s/%s" % (builddir, tarball_dir))
514 run("chmod -R u+w %s/%s/perl-module" % (builddir, tarball_dir))
515 run("mkdir %s/%s/_build" % (builddir, tarball_dir))
516 run("chmod a-w %s/%s" % (builddir, tarball_dir))
518 "cd %(builddir)s/%(tarball_dir)s/_build && ../configure "
519 "--%(perl)s-perl-module --enable-relocatable --prefix='' "
520 "PKG_CONFIG_PATH=$PWD/../../../source/%(ssw_dir)s/inst/lib/pkgconfig "
521 "CPPFLAGS=\"-I$PWD/../../../source/%(ssw_dir)s/inst/include\" "
522 "LDFLAGS=\"-L$PWD/../../../source/%(ssw_dir)s/inst/lib\" "
523 % {"builddir": builddir,
524 "tarball_dir": tarball_dir,
526 "perl": "with" if build_perl else "without"},
528 for basename in ("config.h", "config.log"):
529 save_result_if_exists("build configuration",
530 "%s/%s/_build/%s" % (builddir, tarball_dir,
536 run("cd %s/%s/_build && make -j$(nproc)" % (builddir, tarball_dir), "build")
538 run("cd %s/%s/_build/perl-module && perl Makefile.PL && make -j$(nproc)"
539 % (builddir, tarball_dir), "build Perl module")
541 start_step("Install")
542 run("cd %s/%s/_build && make -j$(nproc) install DESTDIR=$PWD/pspp-%s"
543 % (builddir, tarball_dir, binary_version), "install")
544 run("cd ../source/%s && make -j$(nproc) install DESTDIR=%s/%s/_build/pspp-%s"
545 % (ssw_dir, builddir, tarball_dir, binary_version))
547 run("cd %s/%s/_build/perl-module && "
548 "make -j$(nproc) install DESTDIR=%s/%s/_build/pspp-%s"
549 % (builddir, tarball_dir, builddir, tarball_dir, binary_version),
550 "install Perl module")
551 run("cd %s/%s/_build/perl-module && "
552 "make -j$(nproc) install DESTDIR=$PWD/inst"
553 % (builddir, tarball_dir))
555 start_step("Make binary distribution")
556 run("cd %s/%s/_build && tar cfz pspp-%s.tar.gz pspp-%s"
557 % (builddir, tarball_dir, binary_version, binary_version))
558 save_result("binary distribution", "%s/%s/_build/pspp-%s.tar.gz"
559 % (builddir, tarball_dir, binary_version), 1)
562 ok = try_run("cd %s/%s/_build && make check TESTSUITEFLAGS=-j$(nproc)"
563 % (builddir, tarball_dir), "check")
564 for basename in ("tests/testsuite.log", "tests/testsuite.dir"):
565 save_result_if_exists("test logs", "%s/%s/_build/%s"
566 % (builddir, tarball_dir, basename))
570 start_step("Uninstall")
571 run("cd ../source/%s && make -j$(nproc) uninstall DESTDIR=%s/%s/_build/pspp-%s"
572 % (ssw_dir, builddir, tarball_dir, binary_version))
573 run("cd %s/%s/_build && make -j$(nproc) uninstall DESTDIR=$PWD/pspp-%s"
574 % (builddir, tarball_dir, binary_version), "uninstall")
576 start_step("Check uninstall")
578 run("(cd %s/%s/_build/perl-module/inst && find -type f -print) | "
579 "(cd %s/%s/_build/pspp-%s && xargs rm)"
580 % (builddir, tarball_dir,
581 builddir, tarball_dir, binary_version), "uninstall Perl module")
582 run("cd %s/%s/_build && "
583 "make -j$(nproc) distuninstallcheck distuninstallcheck_dir=$PWD/pspp-%s"
584 % (builddir, tarball_dir, binary_version),
585 "distuninstallcheck")
589 start_step("Success")
590 set_var("result", "success")