[Treepkg-commits] r377 - in branches/treepkg-status: . bin treepkg
scm-commit@wald.intevation.org
scm-commit at wald.intevation.org
Mon Jul 26 14:58:50 CEST 2010
Author: bricks
Date: 2010-07-26 14:58:50 +0200 (Mon, 26 Jul 2010)
New Revision: 377
Modified:
branches/treepkg-status/bin/publishdebianpackages.py
branches/treepkg-status/demopublishpackages.cfg
branches/treepkg-status/treepkg/publish.py
Log:
incremental copying only changed debian packages
Modified: branches/treepkg-status/bin/publishdebianpackages.py
===================================================================
--- branches/treepkg-status/bin/publishdebianpackages.py 2010-07-26 10:12:40 UTC (rev 376)
+++ branches/treepkg-status/bin/publishdebianpackages.py 2010-07-26 12:58:50 UTC (rev 377)
@@ -29,11 +29,13 @@
config_desc = ["distribution", "section", "num_newest",
"build_user", "build_host", "build_listpackages",
- "publish_user", "publish_host", "cachedb",
+ "publish_user", "publish_host",
("architectures", shlex.split, "armel i386 source"),
("after_upload_hook", shlex.split),
("publish_remove_old_packages", convert_bool),
("publish_dir", remove_trailing_slashes),
+ ("cachedb",
+ lambda s: expand_filename(remove_trailing_slashes(s))),
("cachedir",
lambda s: expand_filename(remove_trailing_slashes(s)))]
@@ -106,31 +108,36 @@
if a.trackname > b.trackname: return +1
return cmp(a.arch, b.arch)
-def copy_files_to_destdir(destdir, files, quiet):
+def copy_files_to_destdir(destdir, files, variables, quiet = False):
scp_flags = []
if quiet:
scp_flags.append("-q")
if not os.path.exists(destdir):
os.makedirs(destdir)
+ if files:
+ if variables["build_host"]:
+ userhost = "%(build_user)s@%(build_host)s:" % variables
+ files = [userhost + filename for filename in files]
+ # scp the packages to the cache dir
+ call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files,
+ scp_flags=scp_flags, cachedir=destdir))
- # scp the packages to the cache dir
- call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files,
- scp_flags=scp_flags, cachedir=destdir))
-
-def remove_old_packages(cachedb, newpackages):
+def remove_old_packages(cachedb, newpackages, quiet):
newfiles = [package.filename for package in newpackages]
oldpackages = cachedb.get_old_packages(newfiles)
for package in oldpackages:
# better check if the file really exists
if os.path.isfile(package.filename):
+ if not quiet:
+ print "removing file %s" % package.filename
os.remove(package.filename)
cachedb.remove_packages(oldpackages)
-def copy_packages_to_destdir(cachedb, dir, packages, quiet = False):
+def copy_packages_to_destdir(cachedb, dir, packages, variables, quiet = False):
packages.sort(cmp=sort_trackname_arch)
package = packages[0]
- trackname = package.name
+ trackname = package.trackname
arch = package.arch
destdir = os.path.join(dir, arch, trackname)
files = []
@@ -138,7 +145,7 @@
cachedb.add_package(package)
if package.trackname != trackname or \
package.arch != arch:
- copy_files_to_destdir(destdir, files, quiet)
+ copy_files_to_destdir(destdir, files, variables, quiet)
trackname = package.trackname
arch = package.arch
destdir = os.path.join(dir, arch, trackname)
@@ -146,12 +153,17 @@
# add only to copy files list if the packages differ
if check_package_is_new(package.name, destdir, package.md5sum):
files.append(package.sourcepath)
- print "copy package: %s, source: %s, destdir: %s" % (package.name,
- package.sourcepath, destdir)
-
+ if not quiet:
+ print "copy new file: %s" % package.name
+ elif not quiet:
+ print "file %s did not changed." % package.name
+ copy_files_to_destdir(destdir, files, variables, quiet)
+
def copy_to_cachedir(variables, track, revision, quiet = False, architectures=None):
cachedir = variables["cachedir"]
cachdebfilename = variables["cachedb"]
+ if not quiet:
+ print "using cachedb: %s" % cachdebfilename
cachedb = CacheDb(cachdebfilename)
newpackages = []
treepkginfo = get_treepkg_info(variables)
@@ -199,13 +211,13 @@
binallarchs = (allowedarchs & allarchs) - sourcearch
for packageinfo in binaryallpackages:
for arch in binallarchs:
- filename = os.path.join(cachedir, arch, track.name,
+ filename = os.path.join(cachedir, arch, packageinfo.trackname,
packageinfo.name)
newpackage = Package(filename, packageinfo.trackname, packageinfo.name,
packageinfo.path, arch, get_md5sum(packageinfo))
newpackages.append(newpackage)
- copy_packages_to_destdir(cachedb, cachedir, newpackages, quiet)
- remove_old_packages(cachedb, newpackages)
+ copy_packages_to_destdir(cachedb, cachedir, newpackages, variables, quiet)
+ remove_old_packages(cachedb, newpackages, quiet)
def publish_packages(config_filename, track, revision, dist, section, quiet):
config = read_config(config_filename)
@@ -215,16 +227,15 @@
if section is None:
section = config["section"]
- if not "cachedb" in config.keys():
- print >>sys.stderr, "Error. Please specifiy a cachedb in your config file"
- sys.exit(1)
-
architectures = config["architectures"]
copy_to_cachedir(config, track, revision, quiet, architectures)
-# copy_to_publishdir(config, dist, section, arch, quiet)
+ copy_to_publishdir(config, dist, section, None, quiet)
# update apt archive
-# call(config["after_upload_hook"])
+ if config["after_upload_hook"]:
+ if not quiet:
+ print "running after upload hook"
+ call(config["after_upload_hook"])
def main():
options, args = parse_commandline()
Modified: branches/treepkg-status/demopublishpackages.cfg
===================================================================
--- branches/treepkg-status/demopublishpackages.cfg 2010-07-26 10:12:40 UTC (rev 376)
+++ branches/treepkg-status/demopublishpackages.cfg 2010-07-26 12:58:50 UTC (rev 377)
@@ -59,3 +59,4 @@
# ~user at the beginning of the value and environment variable
# references of the form $VAR or ${VAR} are expanded.
cachedir: /tmp/${LOGNAME}/treepkg-package-cache/%(build_host)s
+cachedb: /tmp/${LOGNAME}/treepkg-package-cache/%(build_host)s-cache.db
Modified: branches/treepkg-status/treepkg/publish.py
===================================================================
--- branches/treepkg-status/treepkg/publish.py 2010-07-26 10:12:40 UTC (rev 376)
+++ branches/treepkg-status/treepkg/publish.py 2010-07-26 12:58:50 UTC (rev 377)
@@ -32,8 +32,11 @@
prefix.extend(["ssh", "%s@%s" % (user, host)])
return prefix
-def copy_to_publishdir(variables, dist, section, arch, quiet):
- destdir = os.path.join(variables["publish_dir"], dist, section, arch)
+def copy_to_publishdir(variables, dist, section, arch=None, quiet=False):
+ if not arch:
+ destdir = os.path.join(variables["publish_dir"], dist, section)
+ else:
+ destdir = os.path.join(variables["publish_dir"], dist, section, arch)
remote_destdir = destdir
if variables["publish_host"]:
remote_destdir = (("%(publish_user)s@%(publish_host)s:" % variables)
More information about the Treepkg-commits
mailing list