[Treepkg-commits] r182 - in trunk: . bin
scm-commit@wald.intevation.org
scm-commit at wald.intevation.org
Thu Apr 16 11:32:41 CEST 2009
Author: bh
Date: 2009-04-16 11:32:35 +0200 (Thu, 16 Apr 2009)
New Revision: 182
Modified:
trunk/bin/publishpackages.py
trunk/demopublishpackages.cfg
Log:
Make publishpackages.py work for automated regular publishing of the
newest built packages. This requires changes in the configuration file
as well, so demopublishpackages.cfg is updated too.
Modified: trunk/bin/publishpackages.py
===================================================================
--- trunk/bin/publishpackages.py 2009-04-15 13:50:31 UTC (rev 181)
+++ trunk/bin/publishpackages.py 2009-04-16 09:32:35 UTC (rev 182)
@@ -11,11 +11,12 @@
import sys
import os
import shutil
+import shlex
from optparse import OptionParser
from ConfigParser import SafeConfigParser
import treepkgcmd
-from treepkg.readconfig import read_config_section
+from treepkg.readconfig import read_config_section, convert_bool
from treepkg.run import call, capture_output
from treepkg.cmdexpand import cmdexpand
from treepkg.util import ensure_directory, listdir_abs
@@ -30,9 +31,11 @@
"""
return os.path.expandvars(os.path.expanduser(filename))
-config_desc = ["distribution", "section",
+config_desc = ["distribution", "section", "num_newest",
"build_user", "build_host", "build_listpackages",
- "publish_user", "publish_host", "publish_apt_archive_update",
+ "publish_user", "publish_host",
+ ("after_upload_hook", shlex.split),
+ ("publish_remove_old_packages", convert_bool),
("publish_dir", remove_trailing_slashes),
("cachedir",
lambda s: expand_filename(remove_trailing_slashes(s)))]
@@ -67,9 +70,19 @@
" will be published"))
return parser.parse_args()
+def prefix_for_remote_command(user, host):
+ """Returns the ssh call needed to run a command on a remote host.
+ If host is empty, the function assumes the command is to be run on
+ the local host as the same user that exectutes this function, so
+ that no ssh or other call is needed.
+ """
+ prefix = []
+ if host:
+ prefix.extend(["ssh", "%s@%s" % (user, host)])
+ return prefix
-def publish_packages_arch(variables, track, revision, dist, section, arch):
- # create web-page on build host
+
+def copy_to_cache(variables, track, revision, arch):
listpackages_vars = variables.copy()
if arch == "source":
@@ -87,30 +100,48 @@
else:
listpackages_vars["revision"] = []
- files = capture_output(cmdexpand("ssh $build_user$@$build_host"
- " $build_listpackages"
- " @track @revision $pkgtype",
+ runremote = prefix_for_remote_command(variables["build_user"],
+ variables["build_host"])
+ files = capture_output(cmdexpand("@runremote $build_listpackages"
+ " @track @revision $pkgtype"
+ " --newest=$num_newest",
+ runremote=runremote,
**listpackages_vars)).strip().split("\n")
# scp the packages to the cache dir
cachedir = variables["cachedir"]
shutil.rmtree(cachedir, ignore_errors=True)
ensure_directory(cachedir)
- userhost = "%(build_user)s@%(build_host)s:" % variables
- call(cmdexpand("scp @files $cachedir/",
- files = [userhost + filename for filename in files],
- **variables))
+ if variables["build_host"]:
+ userhost = "%(build_user)s@%(build_host)s:" % variables
+ files = [userhost + filename for filename in files]
+ call(cmdexpand("scp -p @files $cachedir/", files = files, **variables))
- # copy the packages to the remote publishing host. Create the
- # destination directory if it doesn't exist yet.
+
+def copy_to_publishdir(variables, dist, section, arch):
destdir = os.path.join(variables["publish_dir"], dist, section, arch)
- call(cmdexpand("ssh $publish_user$@$publish_host mkdir --parents $destdir",
- destdir=destdir, **variables))
- call(cmdexpand("scp @files $publish_user$@$publish_host:$destdir",
- files=listdir_abs(cachedir), destdir=destdir,
+ remote_destdir = destdir
+ if variables["publish_host"]:
+ remote_destdir = (("%(publish_user)s@%(publish_host)s:" % variables)
+ + remote_destdir)
+ runremote = prefix_for_remote_command(variables["publish_user"],
+ variables["publish_host"])
+
+ call(cmdexpand("@runremote mkdir --parents $destdir",
+ runremote=runremote, destdir=destdir, **variables))
+ rsync_flags = []
+ if variables["publish_remove_old_packages"]:
+ rsync_flags.append("--delete")
+ call(cmdexpand("rsync @rsync_flags -rpt $cachedir/ $remote_destdir/",
+ rsync_flags=rsync_flags, remote_destdir=remote_destdir,
**variables))
+def publish_packages_arch(variables, track, revision, dist, section, arch):
+ copy_to_cache(variables, track, revision, arch)
+ copy_to_publishdir(variables, dist, section, arch)
+
+
def publish_packages(config_filename, track, revision, dist, section):
config = read_config(config_filename)
@@ -123,9 +154,7 @@
publish_packages_arch(config, track, revision, dist, section, arch)
# update apt archive
- call(cmdexpand("ssh $publish_user$@$publish_host"
- " $publish_apt_archive_update",
- **config))
+ call(config["after_upload_hook"])
def main():
options, args = parse_commandline()
Modified: trunk/demopublishpackages.cfg
===================================================================
--- trunk/demopublishpackages.cfg 2009-04-15 13:50:31 UTC (rev 181)
+++ trunk/demopublishpackages.cfg 2009-04-16 09:32:35 UTC (rev 182)
@@ -9,6 +9,11 @@
distribution: lenny
section: experimental
+# Number of newest successfully builts revisions to publish for each
+# track. Ignored when publishpackages is called with an explicit
+# revision.
+num_newest: 3
+
# Username and host on which the treepackager runs. publishpackages.py
# has to be able to connect to that host as the builduser via ssh
# without knowning the password. This is best achieved with the
@@ -23,17 +28,26 @@
# Username and host on which to publish the packages.
# publishpackages.py uses ssh to execute commands on the publish_host to
-# create directories and scp to copy the files from the local cache to
-# the publish_host.
+# create directories and rsync to copy the files from the local cache to
+# the publish_host. If publish_host is empty, the publishpackages.py
+# assumes that no ssh or other remote login is needed to access the
+# publish host with the right permissions.
publish_user: builder
publish_host: localhost
# the directory on the publish_host where the apt directories reside.
publish_dir: /home/ftp/apt/dists/
-# Command to execute on the build host to update the apt archive
-publish_apt_archive_update: /home/ftp/apt/update
+# If true publishpackages.py will remove the packages in publish_dir
+# that were not put there by the current run. Setting it to True is
+# probably only useful when using publishpackages.py to automatically
+# publish the newest built packages using a cron-job, for instance.
+publish_remove_old_packages: False
+# Command to run after the packages have been copied to the publish
+# host. This command is run on the host running publishpackages.py.
+after_upload_hook: ssh %(publish_user)s@%(publish_host)s /home/ftp/apt/update
+
# local cache directory. publishpackages.py may delete it and its
# contents, so if you intend to run publishpackages.py on the same
# system for several remote instances of treepkg, make sure the
More information about the Treepkg-commits
mailing list