1 #! /usr/bin/env python3
10 from datetime import datetime
11 from pathlib import Path
14 def backquotes(command):
15 output = subprocess.check_output(command, shell=True, text=True)
16 if output.endswith('\n'):
23 %s, for building and testing PSPP
24 usage: %s [OPTIONS] [TARBALL | REPO REFSPEC]
25 where TARBALL is the name of a tarball produced by "make dist"
26 or REPO and REFSPEC are a Git repo and refspec (e.g. branch) to clone.
29 --help Print this usage message and exit
30 --no-binary Build source tarballs but no binaries.
31 --batch Do not print progress to stdout."""
32 % (sys.argv[0], sys.argv[0]))
36 def run(command, id=None):
37 if not try_run(command, id):
41 def try_run(command, id=None):
42 LOG.write("%s\n" % command)
44 p = subprocess.Popen(command, shell=True, text=True,
45 stdout=subprocess.PIPE,
46 stderr=subprocess.STDOUT)
48 start = datetime.now()
49 est_time = 0 if id is None else read_timing(id)
55 elapsed = (datetime.now() - start).seconds
56 progress = "%d lines logged, %d s elapsed" % (lines, elapsed)
59 left = est_time - elapsed
61 progress += ", ETA %d s" % left
63 sys.stdout.write("\r%s%s\r"
64 % (progress, " " * (79 - len(progress))))
66 sys.stdout.write("\r%s\r" % (' ' * 79))
69 write_timing(id, (datetime.now() - start).seconds)
76 sys.stderr.write("%s: child died with signal %d\n" % (command, -rc))
78 sys.stderr.write("%s: child exited with value %d\n" % (command, rc))
84 for s in open("%s/timings" % topdir):
85 m = re.match('([^=]+)=(.*)', s.rstrip())
87 key, value = m.groups()
94 except FileNotFoundError:
98 def write_timing(id, time):
99 tmpname = "%s/timings.tmp%d" % (topdir, os.getpid())
100 NEWTIMINGS = open(tmpname, 'w')
102 OLDTIMINGS = open("%s/timings" % topdir, "r")
104 m = re.match('([^=]+)=(.*)', s.rstrip())
106 key, value = m.groups()
110 except FileNotFoundError:
112 NEWTIMINGS.write("%s=%s\n" % (id, time))
114 os.rename(tmpname, "%s/timings" % topdir)
118 sys.stderr.write("Build failed, refer to:\n\t%s\nfor details.\n" % logfile)
123 LOG.write("
\f\n%s\n" % msg)
128 def set_var(var, value):
129 VARS.write("%s=%s\n" % (var, value))
131 print("\t%s=%s" % (var, value))
133 VAR = open("%s/vars/%s" % (resultsdir, var), "wt")
134 VAR.write("%s\n" % value)
138 def saved_result(name, product):
139 start_step("Saving %s: %s" % (name, product))
142 def save_result(name, src, rm_src=False):
143 basename = os.path.basename(src)
144 dst = "%s/%s" % (resultsdir, basename)
146 saved_result(name, basename)
147 run("cp -R %s %s" % (src, dst))
155 def save_result_if_exists(name, src, rm_src=False):
156 if Path(src).exists():
157 save_result(name, src, rm_src)
159 start_step("%s does not exist, cannot save" % src)
162 def ref_to_commit(ref, repo='.'):
163 return backquotes("cd %s && %s rev-parse %s" % (repo, GIT, ref))
167 opts, args = getopt.gnu_getopt(sys.argv[1:], "ho:",
169 "binary", "no-binary",
172 "builder=", "build-number="])
173 except getopt.GetoptError as err:
174 # print help information and exit:
175 print(err) # will print something like "option -a not recognized"
179 batch = not os.isatty(1)
184 if o in ("-h", "--help"):
186 elif o == "--binary":
188 elif o == "--no-binary":
192 elif o == "--no-batch":
194 elif o in ("-o", "--output"):
196 elif o == "--builder":
198 elif o == "--build-number":
201 assert False, "unhandled option"
203 builder = socket.gethostname()
205 if len(args) not in (1, 2):
207 "%s: exactly one or two nonoption arguments are required\n"
212 tarball = os.path.abspath(args[0])
214 pass # Tarball will be generated later.
216 # Select build number.
217 if build_number is None:
218 build_number = datetime.now().strftime("%Y%m%d%H%M%S")
220 topdir = os.path.dirname(sys.argv[0])
221 if not topdir.startswith("/"):
222 topdir = "%s/%s" % (os.getcwd(), topdir)
224 # Create build directory.
226 builddir = "builds/%s" % build_number
227 if not Path('builds').is_dir():
229 builddir = os.path.abspath(builddir)
230 if not Path(builddir).is_dir():
234 resultsdir = "%s/results" % builddir
236 os.mkdir("%s/vars" % resultsdir)
238 varsfile = "%s/VARS" % resultsdir
239 VARS = open(varsfile, "wt", buffering=1)
241 logfile = "%s/LOG" % resultsdir
242 LOG = open(logfile, "wt", buffering=1)
244 set_var("builder", builder)
245 set_var("build_number", build_number)
247 GIT = "git --git-dir=%s/.git" % topdir
249 # ssw = "https://alpha.gnu.org/gnu/ssw/spread-sheet-widget-0.4.tar.gz"
250 ssw = "https://git.savannah.gnu.org/git/ssw.git master"
252 if ssw.endswith('.tar.gz'):
253 ssw_basename = os.path.basename(ssw)
254 ssw_file = '%s/%s' % (topdir, ssw_basename)
255 ssw_dir = ssw_basename[:-7]
261 if ssw.endswith('.tar.gz'):
262 if not Path(ssw_file).exists():
263 start_step("Retrieve spread-sheet-widget tarball %s" % ssw_file)
264 run("wget -O %s %s" % (ssw_file, ssw))
266 start_step("Extract %s into %s" % (ssw_file, ssw_dir))
267 run("tar xzf %s" % ssw_file)
269 ssw_url, ssw_ref = ssw.split()
271 start_step("Clone spread-sheet-widget into %s" % ssw_dir)
272 run("git clone %s %s" % (ssw_url, ssw_dir))
274 start_step("Check out %s in ssw" % ssw_ref)
275 run("cd %s && git checkout %s" % (ssw_dir, ssw_ref))
276 set_var("ssw_commit", ref_to_commit("HEAD", "ssw"))
278 start_step("Bootstrap ssw")
279 run("cd %s && ./bootstrap" % ssw_dir)
283 start_step("Configure spread-sheet-widget")
284 run("cd %s && ./configure --prefix=''" % ssw_dir)
286 start_step("Build spread-sheet-widget")
287 run("cd %s && make -j$(nproc)" % ssw_dir)
289 start_step("Install spread-sheet-widget")
290 run("cd %s && make -j$(nproc) install DESTDIR=$PWD/inst" % ssw_dir)
292 start_step("Fetch branch from Git")
293 set_var("git_repo", repo)
294 set_var("git_branch", branch)
295 run("%s fetch %s +%s:refs/builds/%s/pspp"
296 % (GIT, repo, branch, build_number))
298 # Get revision number.
299 set_var("pspp_ref", "refs/builds/%s/pspp" % build_number)
300 revision = ref_to_commit("refs/builds/%s/pspp" % build_number)
301 set_var("pspp_commit", revision)
302 abbrev_commit = revision[:6]
305 start_step("Extract branch into source directory")
306 run("%s archive --format=tar --prefix=pspp/ refs/builds/%s/pspp | tar xf -"
307 % (GIT, build_number))
309 # Extract version number.
310 start_step("Extract repository version number")
311 fields = backquotes("cd pspp && autoconf -t AC_INIT").split(':')
312 file, line, macro, package, repo_version = fields[:5]
314 set_var("repo_version", repo_version)
316 # Is this a "gnits" mode tree?
317 start_step("Checking Automake mode")
320 for s in open("pspp/Makefile.am"):
324 LOG.write("Automake mode is %s\n" % am_mode)
326 # Generate version number for build.
327 # We want to append -g012345, but if we're in Gnits mode and the
328 # version number already has a hyphen, we have to omit it.
329 start_step("Generate build version number")
330 version = repo_version
331 if '-' not in version:
333 version += 'g' + abbrev_commit
334 set_var("version", version)
336 # Append -g012345 to configure.ac version number.
337 start_step("Updating version number in %s" % file)
338 fullname = "pspp/%s" % file
339 NEWFILE = open("%s.new" % fullname, "w")
341 for s in open(fullname):
345 NEWFILE.write("AC_INIT([%s], [%s]" % (package, version))
347 NEWFILE.write(", [%s]" % field)
351 os.rename("%s.new" % fullname, fullname)
353 # Get Gnulib commit number.
354 start_step("Reading README.Git to find Gnulib commit number")
355 fullname = "pspp/README.Git"
357 for s in open(fullname):
358 m = re.match(r'\s+commit ([0-9a-fA-F]{8,})', s)
360 gnulib_commit = m.group(1)
362 if gnulib_commit is None:
363 sys.stderr.write("%s does not specify a Git commit number\n"
366 set_var("gnulib_commit", gnulib_commit)
368 # Add note to beginning of NEWS (otherwise "make dist" fails).
369 start_step("Updating NEWS")
370 fullname = "pspp/NEWS"
371 NEWFILE = open("%s.new" % fullname, "w")
372 found_changes = False
373 for s in open(fullname):
374 if not found_changes and s.startswith('Changes'):
377 Changes from %(repo_version)s to %(version)s:
379 * Built from PSPP commit %(revision)s
380 in branch %(branch)s on builder %(builder)s.
382 * Built from Gnulib commit %(gnulib_commit)s.
385 % {'repo_version': repo_version,
387 'revision': revision,
390 'gnulib_commit': gnulib_commit})
394 os.rename("%s.new" % fullname, fullname)
396 # If we don't already have that Gnulib commit, update Gnulib.
397 rc = os.system("%s rev-parse --verify --quiet %s^0 > /dev/null"
398 % (GIT, gnulib_commit))
400 start_step("Updating Gnulib to obtain commit")
401 run("%s fetch gnulib" % GIT)
402 run("%s update-ref refs/builds/%s/gnulib %s"
403 % (GIT, build_number, gnulib_commit))
404 set_var("gnulib_ref", "refs/builds/%s/gnulib" % build_number)
406 # Extract gnulib source.
407 start_step("Extract Gnulib source")
408 run("%s archive --format=tar --prefix=gnulib/ %s | tar xf -"
409 % (GIT, gnulib_commit))
412 start_step("Bootstrap (make -f Smake)")
413 run("cd pspp && make -f Smake -j$(nproc)", "bootstrap")
416 start_step("Configure source")
419 "cd _build && ../configure "
420 "PKG_CONFIG_PATH=$PWD/../../%s/inst/lib/pkgconfig" % ssw_dir,
424 start_step("Make source tarball")
425 run("cd pspp/_build && make -j$(nproc) dist", "dist")
426 tarname = "pspp-%s.tar.gz" % version
427 tarball = save_result("source distribution", "pspp/_build/%s" % tarname, 1)
429 # Save translation templates.
430 potfile = "pspp/_build/po/pspp.pot"
431 if not Path(potfile).exists():
432 potfile = "pspp/po/pspp.pot"
433 save_result("translation templates", potfile)
435 # Build examples for user manual.
436 start_step("Build examples for user manual")
437 run("cd pspp/_build && make -j$(nproc) figure-spvs figure-txts figure-texis figure-htmls")
440 start_step("Build user manual")
442 "GENDOCS_TEMPLATE_DIR=%s %s/gendocs.sh -s doc/pspp.texi -I doc "
443 "-I _build/doc -o %s/user-manual --email bug-gnu-pspp@gnu.org "
444 "pspp \"GNU PSPP User Manual\"" % (topdir, topdir, resultsdir),
446 saved_result("User Manual", "user-manual")
448 # Build developer's guide
449 start_step("Build developers guide")
451 "GENDOCS_TEMPLATE_DIR=%s %s/gendocs.sh -s doc/pspp-dev.texi "
452 "-I doc -o %s/dev-guide --email bug-gnu-pspp@gnu.org "
453 "pspp-dev \"GNU PSPP Developers Guide\""
454 % (topdir, topdir, resultsdir), "dev-guide")
455 saved_result("Developers Guide", "dev-guide")
457 start_step("Starting from %s" % tarball)
460 start_step("Save tarball to Git")
461 run("GIT_DIR=%s/.git %s/git-import-tar %s refs/builds/%s/dist"
462 % (topdir, topdir, tarball, build_number), "git-dist")
463 set_var("dist_ref", "refs/builds/%s/dist" % build_number)
464 set_var("dist_commit", ref_to_commit("refs/builds/%s/dist" % build_number))
466 start_step("Determining %s target directory" % tarball)
467 sample_filename = backquotes("zcat %s | tar tf - | head -1" % tarball)
468 tarball_dir = re.match('(?:[./])*([^/]+)/', sample_filename).group(1)
469 set_var("dist_dir", tarball_dir)
471 start_step("Extracting source tarball")
472 run("zcat %s | (cd %s && tar xf -)" % (tarball, builddir))
474 start_step("Extracting tar version")
475 version_line = backquotes("cd %s/%s && ./configure --version | head -1"
476 % (builddir, tarball_dir))
477 version = re.search(r'configure (\S+)$', version_line).group(1)
478 set_var("dist_version", version)
479 binary_version = "%s-%s-build%s" % (version, builder, build_number)
480 set_var("binary_version", binary_version)
482 start_step("Configuring")
483 run("chmod -R a-w %s/%s" % (builddir, tarball_dir))
484 run("chmod u+w %s/%s" % (builddir, tarball_dir))
485 run("chmod -R u+w %s/%s/perl-module" % (builddir, tarball_dir))
486 run("mkdir %s/%s/_build" % (builddir, tarball_dir))
487 run("chmod a-w %s/%s" % (builddir, tarball_dir))
489 "cd %(builddir)s/%(tarball_dir)s/_build && ../configure "
490 "--with-perl-module --enable-relocatable --prefix='' "
491 "PKG_CONFIG_PATH=$PWD/../../../source/%(ssw_dir)s/inst/lib/pkgconfig "
492 "CPPFLAGS=\"-I$PWD/../../../source/%(ssw_dir)s/inst/include\" "
493 "LDFLAGS=\"-L$PWD/../../../source/%(ssw_dir)s/inst/lib\""
494 % {"builddir": builddir,
495 "tarball_dir": tarball_dir,
498 for basename in ("config.h", "config.log"):
499 save_result_if_exists("build configuration",
500 "%s/%s/_build/%s" % (builddir, tarball_dir,
506 run("cd %s/%s/_build && make -j$(nproc)" % (builddir, tarball_dir), "build")
507 run("cd %s/%s/_build/perl-module && perl Makefile.PL && make -j$(nproc)"
508 % (builddir, tarball_dir), "build Perl module")
510 start_step("Install")
511 run("cd %s/%s/_build && make -j$(nproc) install DESTDIR=$PWD/pspp-%s"
512 % (builddir, tarball_dir, binary_version), "install")
513 run("cd ../source/%s && make -j$(nproc) install DESTDIR=%s/%s/_build/pspp-%s"
514 % (ssw_dir, builddir, tarball_dir, binary_version))
515 run("cd %s/%s/_build/perl-module && "
516 "make -j$(nproc) install DESTDIR=%s/%s/_build/pspp-%s"
517 % (builddir, tarball_dir, builddir, tarball_dir, binary_version),
518 "install Perl module")
519 run("cd %s/%s/_build/perl-module && "
520 "make -j$(nproc) install DESTDIR=$PWD/inst"
521 % (builddir, tarball_dir))
523 start_step("Make binary distribution")
524 run("cd %s/%s/_build && tar cfz pspp-%s.tar.gz pspp-%s"
525 % (builddir, tarball_dir, binary_version, binary_version))
526 save_result("binary distribution", "%s/%s/_build/pspp-%s.tar.gz"
527 % (builddir, tarball_dir, binary_version), 1)
530 ok = try_run("cd %s/%s/_build && make check TESTSUITEFLAGS=-j$(nproc)"
531 % (builddir, tarball_dir), "check")
532 for basename in ("tests/testsuite.log", "tests/testsuite.dir"):
533 save_result_if_exists("test logs", "%s/%s/_build/%s"
534 % (builddir, tarball_dir, basename))
538 start_step("Uninstall")
539 run("cd ../source/%s && make -j$(nproc) uninstall DESTDIR=%s/%s/_build/pspp-%s"
540 % (ssw_dir, builddir, tarball_dir, binary_version))
541 run("cd %s/%s/_build && make -j$(nproc) uninstall DESTDIR=$PWD/pspp-%s"
542 % (builddir, tarball_dir, binary_version), "uninstall")
544 start_step("Check uninstall")
545 run("(cd %s/%s/_build/perl-module/inst && find -type f -print) | "
546 "(cd %s/%s/_build/pspp-%s && xargs rm)"
547 % (builddir, tarball_dir,
548 builddir, tarball_dir, binary_version), "uninstall Perl module")
549 run("cd %s/%s/_build && "
550 "make -j$(nproc) distuninstallcheck distuninstallcheck_dir=$PWD/pspp-%s"
551 % (builddir, tarball_dir, binary_version),
552 "distuninstallcheck")
556 start_step("Success")