[Treepkg-commits] r401 - in trunk: . bin test treepkg treepkg/info
scm-commit@wald.intevation.org
scm-commit at wald.intevation.org
Fri Aug 6 15:41:57 CEST 2010
Author: bricks
Date: 2010-08-06 15:41:54 +0200 (Fri, 06 Aug 2010)
New Revision: 401
Modified:
trunk/
trunk/bin/publishdebianpackages.py
trunk/bin/publishpackages.py
trunk/test/test_info.py
trunk/test/test_util.py
trunk/treepkg/info/data.py
trunk/treepkg/info/status.py
trunk/treepkg/info/status.xsd
trunk/treepkg/packager.py
trunk/treepkg/publish.py
trunk/treepkg/util.py
Log:
Merged revisions 388-399 via svnmerge from
svn+ssh://svn.wald.intevation.org/treepkg/branches/treepkg-status
........
r388 | bricks | 2010-07-28 10:38:50 +0200 (Mi, 28 Jul 2010) | 3 lines
fix if statement
upload hook should be called if it's not empty
........
r389 | bricks | 2010-07-28 13:38:00 +0200 (Mi, 28 Jul 2010) | 2 lines
only rsync specified architectures from cachedir to publishdir
........
r390 | bricks | 2010-07-29 14:07:57 +0200 (Do, 29 Jul 2010) | 2 lines
introduced after_copy_hook and made rsync upload to publishdir optional
........
r391 | bricks | 2010-07-29 15:43:39 +0200 (Do, 29 Jul 2010) | 3 lines
publish package must be default to get original behaviour
be more verbose at publishing
........
r392 | bricks | 2010-07-29 16:50:53 +0200 (Do, 29 Jul 2010) | 2 lines
fix handling of showing only successfull builds together with num packages
........
r393 | bricks | 2010-07-29 16:52:42 +0200 (Do, 29 Jul 2010) | 2 lines
fixed typo
........
r394 | bricks | 2010-07-29 16:59:10 +0200 (Do, 29 Jul 2010) | 2 lines
fixed another typo
........
r395 | bricks | 2010-07-29 18:05:54 +0200 (Do, 29 Jul 2010) | 3 lines
fixed another typo
treepkg is really slow with many revisions
........
r396 | bricks | 2010-08-02 12:41:55 +0200 (Mo, 02 Aug 2010) | 2 lines
fixed typos
........
r397 | bricks | 2010-08-02 17:36:12 +0200 (Mo, 02 Aug 2010) | 3 lines
be more error prone in listing different files
not checking if a dir exists caused several errors if a build wasn't successful
........
r398 | bricks | 2010-08-05 18:21:47 +0200 (Do, 05 Aug 2010) | 2 lines
review changes
........
r399 | bricks | 2010-08-06 13:06:08 +0200 (Fr, 06 Aug 2010) | 4 lines
cleanup modules
fix test_info testcases
added testcases for remove_trailingslashes and expand_filename
........
Property changes on: trunk
___________________________________________________________________
Name: svnmerge-integrated
- /branches/treepkg-status:1-343,346-386
+ /branches/treepkg-status:1-343,346-386,388-399
Modified: trunk/bin/publishdebianpackages.py
===================================================================
--- trunk/bin/publishdebianpackages.py 2010-08-06 13:28:47 UTC (rev 400)
+++ trunk/bin/publishdebianpackages.py 2010-08-06 13:41:54 UTC (rev 401)
@@ -10,8 +10,6 @@
"""Publishes selected packages created by treepkg"""
import os
-import os.path
-import re
import sys
import shlex
@@ -22,23 +20,23 @@
from treepkg.readconfig import read_config_section, convert_bool
from treepkg.run import call, capture_output
from treepkg.cmdexpand import cmdexpand
-from treepkg.publish import *
-from treepkg.util import md5sum
+from treepkg.publish import copy_arch_to_publishdir, prefix_for_remote_command
+
+from treepkg.util import md5sum, expand_filename, remove_trailing_slashes
from treepkg.info.status import TreepkgInfo
from treepkg.info.data import Package
from treepkg.info.data import CacheDb
-EMPTY = re.compile(r'\s*')
-
config_desc = ["distribution", "section", "num_newest",
"build_user", "build_host", "build_listpackages",
"publish_user", "publish_host",
("architectures", shlex.split, "armel i386 source"),
- ("after_upload_hook", shlex.split),
+ ("after_upload_hook", shlex.split, ""),
+ ("after_copy_hook", shlex.split, ""),
+ ("publish_packages", convert_bool, "True"),
("publish_remove_old_packages", convert_bool),
("publish_dir", remove_trailing_slashes),
- ("cachedb",
- lambda s: expand_filename(remove_trailing_slashes(s))),
+ ("cachedb", lambda s: expand_filename(s)),
("cachedir",
lambda s: expand_filename(remove_trailing_slashes(s)))]
@@ -78,18 +76,12 @@
runremote = prefix_for_remote_command(variables["build_user"],
variables["build_host"])
xml = capture_output(cmdexpand("@runremote $build_listpackages"
- " --newest=$num_newest"
- " --only-successful",
- runremote=runremote,
- **variables))
+ " --newest=$num_newest"
+ " --only-successful",
+ runremote=runremote,
+ **variables))
return TreepkgInfo.fromxml(xml)
-def get_binary_arch(arch):
- if not arch is None and not arch.startswith("binary") and \
- not arch == "source":
- arch = "binary-" + arch
- return arch
-
def check_package_is_new(packagename, destdir, packagemd5sum):
destpackage = os.path.join(destdir, packagename)
if not os.path.isfile(destpackage):
@@ -107,11 +99,13 @@
return md5sum
def sort_trackname_arch(a, b):
- if a.trackname < b.trackname: return -1
- if a.trackname > b.trackname: return +1
+ if a.trackname < b.trackname:
+ return -1
+ if a.trackname > b.trackname:
+ return +1
return cmp(a.arch, b.arch)
-def copy_files_to_destdir(destdir, files, variables, quiet = False):
+def copy_files_to_destdir(destdir, files, variables, quiet=False):
scp_flags = []
if quiet:
scp_flags.append("-q")
@@ -137,7 +131,7 @@
os.remove(package.filename)
cachedb.remove_packages(oldpackages)
-def copy_packages_to_destdir(cachedb, dir, packages, variables, quiet = False):
+def copy_packages_to_destdir(cachedb, dir, packages, variables, quiet=False):
packages.sort(cmp=sort_trackname_arch)
package = packages[0]
trackname = package.trackname
@@ -146,8 +140,7 @@
files = []
for package in packages:
cachedb.add_package(package)
- if package.trackname != trackname or \
- package.arch != arch:
+ if package.trackname != trackname or package.arch != arch:
copy_files_to_destdir(destdir, files, variables, quiet)
trackname = package.trackname
arch = package.arch
@@ -160,7 +153,8 @@
print "copy new file: %s" % package.name
copy_files_to_destdir(destdir, files, variables, quiet)
-def copy_to_cachedir(variables, track, revision, quiet = False, architectures=None):
+def copy_to_cachedir(variables, track, revision, quiet=False,
+ architectures=()):
cachedir = variables["cachedir"]
cachdebfilename = variables["cachedb"]
if not quiet:
@@ -168,14 +162,10 @@
cachedb = CacheDb(cachdebfilename)
newpackages = []
treepkginfo = get_treepkg_info(variables)
- #allowedarchs = set([]) # contains all wanted architectures (incl. source)
- allarchs = set([]) # contains all present architectures (incl. source)
+ allarchs = set() # contains all present architectures (incl. source)
binaryallpackages = []
# change e.g. armel in binary-armel
- if not architectures is None:
- allowedarchs = set([get_binary_arch(a) for a in architectures])
- else:
- allowedarchs = set([])
+ allowedarchs = set([get_binary_arch(a) for a in architectures])
for track in treepkginfo.tracks:
for rev in track.revisions:
for packageinfo in rev.packages:
@@ -204,9 +194,9 @@
newpackages.append(newpackage)
# copy binary-all packages
sourcearch = set(["source"])
- if len(allowedarchs) == 0:
+ if not allowedarchs:
binallarchs = allarchs - sourcearch
- elif len(allarchs) == 0:
+ elif allarchs:
binallarchs = allowedarchs - sourcearch
else:
binallarchs = (allowedarchs & allarchs) - sourcearch
@@ -214,8 +204,9 @@
for arch in binallarchs:
filename = os.path.join(cachedir, arch, packageinfo.trackname,
packageinfo.name)
- newpackage = Package(filename, packageinfo.trackname, packageinfo.name,
- packageinfo.path, arch, get_md5sum(packageinfo))
+ newpackage = Package(filename, packageinfo.trackname,
+ packageinfo.name, packageinfo.path, arch,
+ get_md5sum(packageinfo))
newpackages.append(newpackage)
copy_packages_to_destdir(cachedb, cachedir, newpackages, variables, quiet)
remove_old_packages(cachedb, newpackages, quiet)
@@ -230,13 +221,22 @@
section = config["section"]
architectures = config["architectures"]
- allarchs = copy_to_cachedir(config, track, revision, quiet, architectures)
- for arch in allarchs:
- copy_to_publishdir(config, dist, section, arch, quiet)
+ copy_to_cachedir(config, track, revision, quiet, architectures)
+ if config["after_copy_hook"]:
+ if not quiet:
+ print "running after copy hook"
+ call(config["after_copy_hook"])
+
+ if config["publish_packages"]:
+ for arch in architectures:
+ if not quiet:
+ print "publish packages for architecture %s" % arch
+ copy_arch_to_publishdir(config, dist, section,
+ get_binary_arch(arch), quiet)
+
# update apt archive
- if not config["after_upload_hook"] or \
- not EMPTY.match(config["after_upload_hook"]):
+ if config["after_upload_hook"]:
if not quiet:
print "running after upload hook"
call(config["after_upload_hook"])
Modified: trunk/bin/publishpackages.py
===================================================================
--- trunk/bin/publishpackages.py 2010-08-06 13:28:47 UTC (rev 400)
+++ trunk/bin/publishpackages.py 2010-08-06 13:41:54 UTC (rev 401)
@@ -19,8 +19,8 @@
from treepkg.readconfig import read_config_section, convert_bool
from treepkg.run import call, capture_output
from treepkg.cmdexpand import cmdexpand
-from treepkg.util import ensure_directory, listdir_abs
-from treepkg.publish import remove_trailing_slashes, expand_filename
+from treepkg.util import ensure_directory, listdir_abs, \
+ remove_trailing_slashes, expand_filename
from treepkg.publish import prefix_for_remote_command, copy_to_publishdir
config_desc = ["distribution", "section", "num_newest",
Modified: trunk/test/test_info.py
===================================================================
--- trunk/test/test_info.py 2010-08-06 13:28:47 UTC (rev 400)
+++ trunk/test/test_info.py 2010-08-06 13:41:54 UTC (rev 401)
@@ -8,18 +8,14 @@
"""Tests for treepkg.status.status"""
import unittest
-import os.path
+import os
import sys
-test_dir = os.path.dirname(__file__)
-sys.path.append(os.path.join(test_dir, os.pardir))
+from treepkg.info.status import TreepkgInfo, TreepkgRootInfo
-from treepkg.info.status import *
-from treepkg.report import get_packager_group
from filesupport import FileTestMixin
+from treepkg.publish import get_binary_arch
-from publishdebianpackages import get_binary_arch
-
class TreepkgInfoTest(unittest.TestCase, FileTestMixin):
config_contents = """\
[DEFAULT]
@@ -83,7 +79,7 @@
xml = dom.toxml()
self.assertEquals("<info><name>testtreepkg</name></info>", xml)
-class TestPublishDebianPackages(unittest.TestCase, FileTestMixin):
+class TestPublish(unittest.TestCase, FileTestMixin):
def test_get_binary_arch(self):
source = get_binary_arch("source")
Modified: trunk/test/test_util.py
===================================================================
--- trunk/test/test_util.py 2010-08-06 13:28:47 UTC (rev 400)
+++ trunk/test/test_util.py 2010-08-06 13:41:54 UTC (rev 401)
@@ -12,9 +12,11 @@
from filesupport import FileTestMixin
-from treepkg.util import replace_in_file, listdir_abs, md5sum
+from treepkg.util import replace_in_file, listdir_abs, md5sum, \
+ remove_trailing_slashes, expand_filename
+
class TestReplaceInFile(unittest.TestCase, FileTestMixin):
def runtest(self, orig_contents, expected_contents, pattern, replacement):
@@ -82,3 +84,21 @@
def test_md5sum(self):
sum = md5sum(self.testfile)
self.assertEquals("a12511153555c1f0f0a1eda200733a3f", sum)
+
+class TestRemoveTrailingSlashes(unittest.TestCase):
+
+ def test_remove_trailing_slashes(self):
+ dir_w_slash = "/tmp/dir/"
+ dir_wo_slash = "/tmp/dir"
+
+ self.assertEquals(dir_wo_slash, remove_trailing_slashes(dir_w_slash))
+ self.assertEquals(dir_wo_slash, remove_trailing_slashes(dir_wo_slash))
+
+class TestExpandFilename(unittest.TestCase):
+
+ def test_expand_filenam(self):
+ os.environ['MY_TEST_VAR'] = "def"
+ path = "/abc/${MY_TEST_VAR}/"
+
+ self.assertEquals("/abc/def/", expand_filename(path))
+
Modified: trunk/treepkg/info/data.py
===================================================================
--- trunk/treepkg/info/data.py 2010-08-06 13:28:47 UTC (rev 400)
+++ trunk/treepkg/info/data.py 2010-08-06 13:41:54 UTC (rev 401)
@@ -15,7 +15,7 @@
class Package:
def __init__(self, filename, trackname, packagename, packagepath,
- arch, md5sum):
+ arch, md5sum):
self.filename = filename
self.trackname = trackname
self.name = packagename
@@ -26,7 +26,6 @@
class CacheDb:
-
def __init__(self, file):
self.SELECT_PACKAGE_TMPL = """SELECT * FROM packages
WHERE filename = ?"""
@@ -77,7 +76,7 @@
row = self.cursor.fetchone()
if not row:
return None
- return Package(row[0], row[1], row[2], row[3], row[4], row[5])
+ return Package(*row)
def get_old_packages(self, newfiles):
SELECT_TMPL = """SELECT * FROM packages
Modified: trunk/treepkg/info/status.py
===================================================================
--- trunk/treepkg/info/status.py 2010-08-06 13:28:47 UTC (rev 400)
+++ trunk/treepkg/info/status.py 2010-08-06 13:41:54 UTC (rev 401)
@@ -45,13 +45,14 @@
if not node:
if required:
raise TreepkgInfoException("Element %s is required as child. But"
- "parent element is not available.")
+ "parent element is not available.")
return None
childs = node.getElementsByTagName(name)
if not childs:
if required:
raise TreepkgInfoException("Element %s is required as child for %s."
- "The XML file must be invalid." % (name, node.nodeName))
+ "The XML file must be invalid."
+ % (name, node.nodeName))
return None
return childs
@@ -66,7 +67,7 @@
self.success = success
group = get_packager_group(config)
treepkgrootinfo = TreepkgRootInfo(group.name, group.treepkg_dir,
- group.tracks_dir)
+ group.tracks_dir)
version = "1.0"
self.tpkgroot = TreepkgRoot(version, treepkgrootinfo)
tracks = group.get_package_tracks()
@@ -91,16 +92,19 @@
revisions = track.get_revisions()
revisions = sorted(revisions, key=lambda r: r.status.start,
reverse=True)
+ candidates = revisions
+
+ if self.success: # skip not successful builds
+ candidates = [rev for rev in revisions
+ if rev.status.status.finished
+ and not rev.status.status.error]
+
if self.numnewestrev > 0:
- revisions = revisions[:self.numnewestrev]
+ candidates = candidates[:self.numnewestrev]
arch = None
- for rev in revisions:
- if self.success: # skip not successful builds
- if not rev.status.status.finished \
- or rev.status.status.error:
- continue
+ for rev in candidates:
revision = rev.revision
rules_revision = rev.rules_revision
status = rev.status.status.description # extend status
@@ -131,8 +135,9 @@
pkginfo.add_checksum(checksuminfo)
revision.add_package(pkginfo)
+
class TreepkgRoot:
-
+
def __init__(self, version, info):
self.version = version
self.info = info
@@ -308,8 +313,6 @@
messageele = getChild(statusele, "message")
message = getTextFromNode(messageele)
treepkgrevisioninfo = TreepkgTrackRevisionInfo(number, rules, message)
- treepkgrevisioninfo.packages = []
- treepkgrevisioninfo.logs = []
packagesele = getChild(node, "packages")
packageeles = getChilds(packagesele, "package")
for packageele in packageeles:
@@ -318,8 +321,7 @@
logsele = getChild(node, "logs")
logeles = getChilds(logsele, "log")
for logele in logeles:
- treepkgrevisioninfo.logs.append(
- TreepkgLogInfo.fromxml(logele))
+ treepkgrevisioninfo.logs.append(TreepkgLogInfo.fromxml(logele))
return treepkgrevisioninfo
class TreepkgLogInfo:
@@ -391,12 +393,13 @@
path = getTextFromNode(pathele)
ptype = node.getAttribute("type")
arch = node.getAttribute("arch")
- if len(arch) == 0:
+ if not arch:
arch = None
packageinfo = TreepkgPackageInfo(name, path, ptype, arch)
checksumeles = node.getElementsByTagName("checksum")
for checksumele in checksumeles:
- packageinfo.checksums.append(TreepkgChecksumInfo.fromxml(checksumele))
+ packageinfo.checksums.append(
+ TreepkgChecksumInfo.fromxml(checksumele))
return packageinfo
class TreepkgChecksumInfo:
@@ -418,6 +421,6 @@
def fromxml(node):
checksum = getTextFromNode(node)
ctype = node.getAttribute("type")
- if len(ctype) == 0:
+ if not ctype:
return TreepkgChecksumInfo(checksum)
return TreepkgChecksumInfo(checksum, ctype)
Modified: trunk/treepkg/info/status.xsd
===================================================================
--- trunk/treepkg/info/status.xsd 2010-08-06 13:28:47 UTC (rev 400)
+++ trunk/treepkg/info/status.xsd 2010-08-06 13:41:54 UTC (rev 401)
@@ -79,7 +79,7 @@
<xsd:complexType name="package">
<xsd:sequence>
<xsd:element name="name" type="xsd:string" minOccurs="1"/>
- <xsd:element name="path" type="xsd:string" minOccurs="1">
+ <xsd:element name="path" type="xsd:string" minOccurs="1"/>
<xsd:element name="checksum" type="tpkg:checksum"
maxOccurs="unbounded" default="md5"/>
</xsd:sequence>
Modified: trunk/treepkg/packager.py
===================================================================
--- trunk/treepkg/packager.py 2010-08-06 13:28:47 UTC (rev 400)
+++ trunk/treepkg/packager.py 2010-08-06 13:41:54 UTC (rev 401)
@@ -8,6 +8,7 @@
"""Classes to automatically build debian packages from subversion checkouts"""
import os
+import os.path
import time
import re
import logging
@@ -327,11 +328,11 @@
files = []
if os.path.isdir(self.log_dir):
for f in os.listdir(self.log_dir):
- if f in logs:
+ if logs is None or f in logs:
f = os.path.join(self.log_dir,f)
if os.path.isfile(f):
files.append((self.get_log_title(f),f))
- return files
+ return files
def list_log_files(self, logs):
"""Returns a list describing the logfiles available for the revision.
@@ -350,14 +351,20 @@
The implementation assumes that all files in self.src_dir belong
to the source package.
"""
- return sorted(util.listdir_abs(self.src_dir))
+ files = []
+ if os.path.isdir(self.src_dir):
+ files = sorted(util.listdir_abs(self.src_dir))
+ return files
def list_binary_files(self):
"""Returns a list with the names of the files of the binary packages.
The implementation assumes that all files in self.binary_dir belong
to the binary packages.
"""
- return sorted(util.listdir_abs(self.binary_dir))
+ files = []
+ if os.path.isdir(self.binary_dir):
+ files = sorted(util.listdir_abs(self.binary_dir))
+ return files
def package(self):
try:
Modified: trunk/treepkg/publish.py
===================================================================
--- trunk/treepkg/publish.py 2010-08-06 13:28:47 UTC (rev 400)
+++ trunk/treepkg/publish.py 2010-08-06 13:41:54 UTC (rev 401)
@@ -9,18 +9,10 @@
import os.path
+import util
from treepkg.run import call, capture_output
from treepkg.cmdexpand import cmdexpand
-def remove_trailing_slashes(s):
- return s.rstrip("/")
-
-def expand_filename(filename):
- """
- Applies os.path.expanduser and os.path.expandvars to filename
- """
- return os.path.expandvars(os.path.expanduser(filename))
-
def prefix_for_remote_command(user, host):
"""Returns the ssh call needed to run a command on a remote host.
If host is empty, the function assumes the command is to be run on
@@ -32,6 +24,46 @@
prefix.extend(["ssh", "%s@%s" % (user, host)])
return prefix
+def copy_arch_to_publishdir(variables, dist, section, arch=None, quiet=False,
+ create=True):
+ if not arch:
+ cachedir = variables["cachedir"]
+ else:
+ cachedir = os.path.join(variables["cachedir"], arch)
+
+ # if cachedir does not exist rsync will fail therefore
+ # it must be created or skipped. if it is created remote
+ # content will be deleted
+ if not os.path.exists(cachedir):
+ if create:
+ util.ensure_directory(cachedir)
+ else:
+ return
+
+ destdir = os.path.join(variables["publish_dir"], dist, section)
+ remote_destdir = destdir
+ if variables["publish_host"]:
+ remote_destdir = (("%(publish_user)s@%(publish_host)s:" % variables)
+ + remote_destdir)
+ runremote = prefix_for_remote_command(variables["publish_user"],
+ variables["publish_host"])
+
+ call(cmdexpand("@runremote mkdir --parents $destdir",
+ runremote=runremote, destdir=destdir, **variables))
+ rsync_flags = []
+ if variables["publish_remove_old_packages"]:
+ rsync_flags.append("--delete")
+ if quiet:
+ rsync_flags.append("--quiet")
+ cmd = cmdexpand("rsync @rsync_flags -r --perms --times --omit-dir-times"
+ " $cachedir $remote_destdir/",
+ rsync_flags=rsync_flags, remote_destdir=remote_destdir,
+ cachedir=cachedir)
+ #print "rsync cmd: %s" % cmd
+ call(cmd)
+
+
+
def copy_to_publishdir(variables, dist, section, arch=None, quiet=False):
if not arch:
destdir = os.path.join(variables["publish_dir"], dist, section)
@@ -56,4 +88,8 @@
rsync_flags=rsync_flags, remote_destdir=remote_destdir,
**variables))
+def get_binary_arch(arch):
+ if not arch is None and not arch.startswith("binary") and arch != "source":
+ arch = "binary-" + arch
+ return arch
Modified: trunk/treepkg/util.py
===================================================================
--- trunk/treepkg/util.py 2010-08-06 13:28:47 UTC (rev 400)
+++ trunk/treepkg/util.py 2010-08-06 13:41:54 UTC (rev 401)
@@ -174,3 +174,13 @@
f.close()
return m.hexdigest()
+def remove_trailing_slashes(s):
+ return s.rstrip("/")
+
+def expand_filename(filename):
+ """
+ Applies os.path.expanduser and os.path.expandvars to filename
+ """
+ return os.path.expandvars(os.path.expanduser(filename))
+
+
More information about the Treepkg-commits
mailing list