[Treepkg-commits] r373 - in branches/treepkg-status: bin test treepkg/info
scm-commit@wald.intevation.org
scm-commit at wald.intevation.org
Fri Jul 23 18:40:39 CEST 2010
Author: bricks
Date: 2010-07-23 18:40:38 +0200 (Fri, 23 Jul 2010)
New Revision: 373
Modified:
branches/treepkg-status/bin/publishdebianpackages.py
branches/treepkg-status/bin/treepkginfo.py
branches/treepkg-status/test/test_info.py
branches/treepkg-status/treepkg/info/status.py
Log:
implemented CacheDb to store copied Packages from build host
Modified: branches/treepkg-status/bin/publishdebianpackages.py
===================================================================
--- branches/treepkg-status/bin/publishdebianpackages.py 2010-07-22 16:01:43 UTC (rev 372)
+++ branches/treepkg-status/bin/publishdebianpackages.py 2010-07-23 16:40:38 UTC (rev 373)
@@ -12,6 +12,7 @@
import os
import sys
import shlex
+import sqlite3
from optparse import OptionParser
from ConfigParser import SafeConfigParser
@@ -34,6 +35,91 @@
("cachedir",
lambda s: expand_filename(remove_trailing_slashes(s)))]
+class Package:
+
+ def __init__(self, filename, trackname, packagename, packagepath, arch, md5sum):
+ self.filename = filename
+ self.trackname = trackname
+ self.name = packagename
+ self.sourcepath = packagepath
+ self.arch = arch
+ self.md5sum = md5sum
+
+
+class CacheDb:
+
+
+ def __init__(self, file):
+ self.SELECT_PACKAGE_TMPL = """SELECT * FROM packages
+ WHERE filename = ?"""
+ self.file = file
+ self.conn = sqlite3.connect(file)
+ self.cursor = self.conn.cursor()
+ self.init_db(file)
+
+ def init_db(self, file):
+ self.cursor.execute(
+ """CREATE TABLE IF NOT EXISTS packages (
+ filename TEXT PRIMARY KEY,
+ trackname TEXT,
+ packagename TEXT,
+ sourcepath TEXT,
+ arch TEXT,
+ md5sum TEXT )""")
+ self.conn.commit()
+
+
+ def add_package(self, package):
+ self.cursor.execute(self.SELECT_PACKAGE_TMPL, (package.filename,))
+ row = self.cursor.fetchone()
+ if not row:
+ self.insert_package(package)
+ else:
+ self.update_package(package)
+
+ def insert_package(self, package):
+ INSERT_TMPL = """INSERT INTO packages VALUES (
+ ?, ?, ? ,? ,?, ?)"""
+ self.cursor.execute(INSERT_TMPL, (package.filename,
+ package.trackname,
+ package.name,
+ package.sourcepath,
+ package.arch,
+ package.md5sum))
+ self.conn.commit()
+
+ def update_package(self, package):
+ UPDATE_TMPL = """UPDATE packages set md5sum = '?'
+ WHERE filename in (?)"""
+ self.cursor.execute(UPDATE_TMPL, (package.md5sum, package.filename))
+ self.conn.commit()
+
+ def get_package(self, filename):
+ self.cursor.execute(self.SELECT_PACKAGE_TMPL, (filename,))
+ row = self.cursor.fetchone()
+ if not row:
+ return None
+ return Package(row[0], row[1], row[2], row[3], row[4], row[5])
+
+ def get_old_packages(self, newfiles):
+ SELECT_TMPL = """SELECT * FROM packages
+ WHERE filename not in (%s)"""
+ tmp = ", ".join(['?'] * len(newfiles))
+ self.cursor.execute(SELECT_TMPL % tmp, newfiles)
+ return [Package(*row) for row in self.cursor.fetchall()]
+
+ def remove_packages(self, files):
+ DELET_TMPL = """DELETE FROM packages
+ WHERE filename in (%s)"""
+ tmp = ", ".join(['?'] * len(files))
+ self.cursor.execute(DELET_TMPL % tmp, files)
+ self.conn.commit()
+
+ def get_packages(self):
+ SELECT_TMPL = "SELECT * FROM packages"
+ self.cursor.execute(SELECT_TMPL)
+ return [Package(*row) for row in self.cursor.fetchall()]
+
def read_config(filename):
if not os.path.exists(filename):
print >>sys.stderr, "Config file %s does not exist" % filename
@@ -70,8 +156,7 @@
variables["build_host"])
xml = capture_output(cmdexpand("@runremote $build_listpackages"
" --newest=$num_newest",
- #runremote=runremote,
- runremote="",
+ runremote=runremote,
**variables))
return TreepkgInfo.fromxml(xml)
@@ -88,26 +173,55 @@
destmd5sum = md5sum(destpackage)
return (destmd5sum != packagemd5sum)
-def copy_to_destdir(package, dir, arch, subdir, quiet = False):
+def get_md5sum(packageinfo):
+ md5sum = ""
+ if packageinfo:
+ for checksum in packageinfo.checksums:
+ if checksum.type == "md5":
+ md5sum = checksum.checksum
+ break
+ return md5sum
+
+def sort_trackname_arch(a, b):
+ if a.trackname < b.trackname: return -1
+ if a.trackname > b.trackname: return +1
+ return cmp(a.arch, b.arch)
+
+def copy_files(destdir, files, quiet):
scp_flags = []
if quiet:
scp_flags.append("-q")
- destdir = os.path.join(dir, arch, subdir)
- print "package: %s, destdir: %s" % (package.name, destdir)
- md5sum = ""
- for checksum in package.checksums:
- if checksum.type == "md5":
- md5sum = checksum.checksum
- break
- # update only if the packages differ
- if check_package_is_new(package.name, destdir, md5sum):
- pass
- # scp the packages to the cache dir
- #call(cmdexpand("scp -p @scp_flags $file $cachedir/", file=package.path,
- # scp_flags=scp_flags, cachedir=destdir))
+ if not os.path.exists(destdir):
+ os.makedirs(destdir)
+
+ # scp the packages to the cache dir
+ call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files,
+ scp_flags=scp_flags, cachedir=destdir))
+
+def copy_to_destdir(dir, packages, quiet = False):
+ packages.sort(cmp=sort_trackname_arch)
+ package = packages[0]
+ trackname = package.name
+ arch = package.arch
+ destdir = os.path.join(dir, arch, trackname)
+ files = []
+ for package in packages:
+ if package.trackname != trackname or \
+ package.arch != arch:
+ #copy_files(destdir, files, quiet)
+ trackname = package.trackname
+ arch = package.arch
+ destdir = os.path.join(dir, arch, trackname)
+ files = []
+ # add only if the packages differ
+ if check_package_is_new(package.name, destdir, package.md5sum):
+ files.append(package.sourcepath)
+ print "package: %s, destdir: %s" % (package.name, destdir)
+
def copy_to_cache(variables, track, revision, quiet = False, architectures=None):
cachedir = variables["cachedir"]
+ newpackages = []
treepkginfo = get_treepkg_info(variables)
#allowedarchs = set([]) # contains all wanted architectures (incl. source)
allarchs = set([]) # contains all present architectures (incl. source)
@@ -117,12 +231,11 @@
allowedarchs = set([get_binary_arch(a) for a in architectures])
else:
allowedarchs = set([])
-
+ print "allowedarchs: %s" % allowedarchs
for track in treepkginfo.tracks:
for rev in track.revisions:
for package in rev.packages:
arch = get_binary_arch(package.arch)
- print "%s %s %s %s" % (track.name, package.name, package.type, package.arch)
if package.type == "binary":
# skip other files
if package.arch is None:
@@ -136,15 +249,15 @@
allarchs.add(arch)
elif package.type == "source":
arch = package.type
+ print "package: %s %s" % (package.name, arch)
# only copy requested archs
if len(allowedarchs) == 0 or \
arch in allowedarchs:
- copy_to_destdir(package, cachedir, arch, track.name,
- quiet)
-
- #print "architectures %s" % architectures
- #print "allowed %s" % allowedarchs
- #print "all %s" % allarchs
+ filename = os.path.join(cachedir, arch, track.name,
+ package.name)
+ newpackage = Package(filename, track.name, package.name,
+ package.path, arch, get_md5sum(package))
+ newpackages.append(newpackage)
# copy binary-all packages
sourcearch = set(["source"])
if len(allowedarchs) == 0:
@@ -153,10 +266,15 @@
binallarchs = allowedarchs - sourcearch
else:
binallarchs = (allowedarchs & allarchs) - sourcearch
- #print "binallarcgs %s" % binallarchs
for package in binaryallpackages:
for arch in binallarchs:
- copy_to_destdir(package, cachedir, arch, package.trackname, quiet)
+ filename = os.path.join(cachedir, arch, track.name,
+ package.name)
+ newpackage = Package(filename, package.trackname, package.name,
+ package.path, arch, get_md5sum(package))
+ newpackages.append(newpackage)
+ print newpackages
+ copy_to_destdir(cachedir, newpackages, quiet)
def publish_packages_arch(variables, track, revision, dist, section,
quiet, architectures):
Modified: branches/treepkg-status/bin/treepkginfo.py
===================================================================
--- branches/treepkg-status/bin/treepkginfo.py 2010-07-22 16:01:43 UTC (rev 372)
+++ branches/treepkg-status/bin/treepkginfo.py 2010-07-23 16:40:38 UTC (rev 373)
@@ -19,6 +19,8 @@
parser.set_defaults(newest=-1)
parser.add_option("--newest", type="int",
help=("Number of newest revisions to list."))
+ parser.add_option("--only-successful", action="store_true",
+ help=("Show only successful builds", dest="success"))
parser.add_option("--pretty", action="store_true", help=("Show XML pretty"
" output."))
@@ -28,7 +30,8 @@
def main():
options, args = parse_commandline()
- tpkginfo = status.TreepkgInfo(options.config_file, options.newest)
+ tpkginfo = status.TreepkgInfo(options.config_file, options.newest,
+ option.success)
if options.pretty:
print tpkginfo.toxml().toprettyxml()
Modified: branches/treepkg-status/test/test_info.py
===================================================================
--- branches/treepkg-status/test/test_info.py 2010-07-22 16:01:43 UTC (rev 372)
+++ branches/treepkg-status/test/test_info.py 2010-07-23 16:40:38 UTC (rev 373)
@@ -13,11 +13,16 @@
test_dir = os.path.dirname(__file__)
sys.path.append(os.path.join(test_dir, os.pardir))
+sys.path.append(os.path.join(test_dir, os.pardir, "bin"))
from treepkg.info.status import *
from treepkg.report import get_packager_group
from filesupport import FileTestMixin
+from publishdebianpackages import get_binary_arch
+from publishdebianpackages import Package
+from publishdebianpackages import CacheDb
+
class TreepkgInfoTest(unittest.TestCase, FileTestMixin):
config_contents = """\
[DEFAULT]
@@ -69,9 +74,8 @@
def test_createinfo(self):
config_file = os.path.join(self.directory, "treepkg.cfg")
tpkginfo = TreepkgInfo(config_file)
- print tpkginfo.toxml().toxml()
- tpkgroot = tpkginfo.tpkgroot
- self.assertEquals("testtreepkginfo", tpkgroot.name)
+ tpkgrootinfo = tpkginfo.tpkgroot.info
+ self.assertEquals("testtreepkginfo", tpkgrootinfo.name)
class TreepkgRootInfoTest(unittest.TestCase):
@@ -80,8 +84,53 @@
status = TreepkgRootInfo("testtreepkg")
dom = status.toxml()
xml = dom.toxml()
- self.assertEquals("<status><name>testtreepkg</name></status>", xml)
+ self.assertEquals("<info><name>testtreepkg</name></info>", xml)
+class TestPublishDebianPackages(unittest.TestCase, FileTestMixin):
+
+ def test_get_binary_arch(self):
+ source = get_binary_arch("source")
+ self.assertEquals("source", source)
+ binary_armel = get_binary_arch("armel")
+ self.assertEquals("binary-armel", binary_armel)
+ binary_armel = get_binary_arch("binary-armel")
+ self.assertEquals("binary-armel", binary_armel)
+
+ def test_cache_db(self):
+ tmpdir = self.create_test_specific_temp_dir()
+ package = Package("/tmp/abc/abc_0.1_i386.deb", "abc",
+ "abc_0.1_i386.deb", "/source/abc/abc_0.1_i386.deb",
+ "binary-i386", "1234567")
+# tmpfile = self.create_temp_file("cachedb1", "abc")
+# with self.assertRaises(Exception):
+# db = CacheDb(tmpfile)
+
+ dbfile = os.path.join(tmpdir, "cachedb2")
+ db = CacheDb(dbfile)
+ db.add_package(package)
+
+ # test get_package and get_timestamp
+ package2 = db.get_package(package.filename)
+ self.assertNotEquals(None, package2)
+ self.assertEquals(package.filename, package2.filename)
+ self.assertEquals(package.name, package2.name)
+ self.assertEquals(package.sourcepath, package2.sourcepath)
+ self.assertEquals(package.arch, package2.arch)
+ self.assertEquals(package.md5sum, package2.md5sum)
+
+ # test get_old_packages
+ package3 = Package("/tmp/foo/foo_0.2.i386.deb", "foo",
+ "foo_0.2_i386.deb", "/tmp/foo/foo_0.2.i386.deb",
+ "binary-i386", "987654321")
+ db.add_package(package3)
+ oldpackages = db.get_old_packages([package.filename])
+ self.assertEquals(1, len(oldpackages))
+ packages = db.get_packages()
+ self.assertEquals(2, len(packages))
+ db.remove_packages([package.filename for package in oldpackages])
+ packages = db.get_packages()
+ self.assertEquals(1, len(packages))
+
if __name__ == '__main__':
unittest.main()
Modified: branches/treepkg-status/treepkg/info/status.py
===================================================================
--- branches/treepkg-status/treepkg/info/status.py 2010-07-22 16:01:43 UTC (rev 372)
+++ branches/treepkg-status/treepkg/info/status.py 2010-07-23 16:40:38 UTC (rev 373)
@@ -61,8 +61,9 @@
class TreepkgInfo:
- def __init__(self, config, numnewestrev=-1):
+ def __init__(self, config, numnewestrev=-1, success=False):
self.numnewestrev = numnewestrev
+ self.success = success
group = get_packager_group(config)
treepkgrootinfo = TreepkgRootInfo(group.name, group.treepkg_dir,
group.tracks_dir)
@@ -96,6 +97,10 @@
arch = None
for rev in revisions:
+ if self.success: # skip not successful builds
+ if not rev.status.status.finished \
+ or rev.status.status.error:
+ continue
revision = rev.revision
rules_revision = rev.rules_revision
status = rev.status.status.description # extend status
More information about the Treepkg-commits
mailing list