[Dive4elements-commits] [PATCH 3 of 3] Importer: Behold, Logging!
Wald Commits
scm-commit at wald.intevation.org
Fri Feb 15 16:22:19 CET 2013
# HG changeset patch
# User Andre Heinecke <aheinecke at intevation.de>
# Date 1360941733 -3600
# Branch dami
# Node ID 769593a846063dc93b0db3dddda7f3b5a5ce5ecc
# Parent eb2d6609387cd82012411a9a04f746f1a1c4af8b
Importer: Behold, Logging!
diff -r eb2d6609387c -r 769593a84606 flys-backend/contrib/shpimporter/hws.py
--- a/flys-backend/contrib/shpimporter/hws.py Fri Feb 15 16:19:52 2013 +0100
+++ b/flys-backend/contrib/shpimporter/hws.py Fri Feb 15 16:22:13 2013 +0100
@@ -9,6 +9,9 @@
from importer import Importer
import utils
+import logging
+logger = logging.getLogger("Hochwasserschutzanlagen")
+
PATH="Hydrologie/HW-Schutzanlagen"
NAME="HWS"
@@ -93,7 +96,7 @@
self.handled(artname)
kind_id = HWS_KIND.get(feat.GetField(artname).lower())
if not kind_id:
- print ("Unknown Art: %s" % \
+ logger.warn("Unknown Art: %s" % \
feat.GetField(artname))
else:
newFeat.SetField("kind_id", kind_id)
@@ -104,7 +107,7 @@
fed_id = FED_STATES.get(feat.GetField(fname).lower())
if not fed_id:
- print ("Unknown Bundesland: %s" % \
+ logger.warn("Unknown Bundesland: %s" % \
feat.GetField("Bundesland"))
else:
newFeat.SetField("fed_state_id", fed_id)
@@ -123,7 +126,7 @@
if self.IsFieldSet(feat, fname):
self.handled(fname)
if feat.GetField(fname) != self.river_id:
- print ("River_id mismatch between shapefile and"
+ logger.warn("River_id mismatch between shapefile and"
" importer parameter.")
newFeat.SetField("river_id", feat.GetField(fname))
else:
diff -r eb2d6609387c -r 769593a84606 flys-backend/contrib/shpimporter/importer.py
--- a/flys-backend/contrib/shpimporter/importer.py Fri Feb 15 16:19:52 2013 +0100
+++ b/flys-backend/contrib/shpimporter/importer.py Fri Feb 15 16:22:13 2013 +0100
@@ -4,6 +4,9 @@
import ogr, osr
import utils
import re
+import logging
+
+logger = logging.getLogger("importer")
class Importer:
@@ -78,20 +81,20 @@
shp = ogr.Open(shape[1])
if shp is None:
- shpimporter.ERROR("Shapefile '%s' could not be opened!" % path)
+ logger.error("Shapefile '%s' could not be opened!" % path)
return
if not self.isShapeRelevant(name, path):
- shpimporter.INFO("Skip shapefile: '%s' of Type: %s" % (path,
+ logger.info("Skip shapefile: '%s' of Type: %s" % (path,
utils.getWkbString(shp.GetLayerByName(name).GetGeomType())))
return
- shpimporter.INFO("Processing shapefile '%s'" % path)
+ logger.info("Processing shapefile '%s'" % path)
srcLayer = shp.GetLayerByName(name)
if srcLayer is None:
- shpimporter.ERROR("Layer '%s' was not found!" % name)
+ logger.error("Layer '%s' was not found!" % name)
return
return self.shape2Database(srcLayer, name, path)
@@ -101,7 +104,7 @@
src_srs = geometry.GetSpatialReference()
if src_srs is None:
- shpimporter.ERROR("No source SRS given! No transformation possible!")
+ logger.error("No source SRS given! No transformation possible!")
return feat
transformer = osr.CoordinateTransformation(src_srs, self.dest_srs)
@@ -158,15 +161,15 @@
destLayer = self.dbconn.GetLayerByName(self.getTablename())
if srcLayer is None:
- shpimporter.ERROR("Shapefile is None!")
+ logger.error("Shapefile is None!")
return -1
if destLayer is None:
- shpimporter.ERROR("No destination layer given!")
+ logger.error("No destination layer given!")
return -1
count = srcLayer.GetFeatureCount()
- shpimporter.DEBUG("Try to add %i features to database." % count)
+ logger.debug("Try to add %i features to database." % count)
srcLayer.ResetReading()
self.srcLayer = srcLayer
@@ -181,7 +184,7 @@
geom = feat.GetGeometryRef()
if geom is None:
- shpimporter.DEBUG("Unkown Geometry reference for feature")
+ logger.debug("Unkown Geometry reference for feature")
continue
geomType = geom.GetGeometryType()
@@ -198,11 +201,11 @@
if newFeat:
res = destLayer.CreateFeature(newFeat)
if res is None or res > 0:
- shpimporter.ERROR("Unable to insert feature. Error: %r" % res)
+ logger.error("Unable to insert feature. Error: %r" % res)
else:
success = success + 1
else:
- shpimporter.ERROR("Could not transform feature: %s " % feat.GetFID())
+ logger.error("Could not transform feature: %s " % feat.GetFID())
creationFailed += 1
else:
creationFailed = creationFailed + 1
@@ -210,10 +213,10 @@
unsupported[utils.getWkbString(geomType)] = \
unsupported.get(utils.getWkbString(geomType), 0) + 1
- shpimporter.INFO("Inserted %i features" % success)
- shpimporter.INFO("Failed to create %i features" % creationFailed)
+ logger.info("Inserted %i features" % success)
+ logger.info("Failed to create %i features" % creationFailed)
for key, value in unsupported.items():
- shpimporter.INFO("Found %i unsupported features of type: %s" % (value, key))
+ logger.info("Found %i unsupported features of type: %s" % (value, key))
if self.tracking_import:
unhandled = []
@@ -223,7 +226,7 @@
unhandled.append(act_field)
if len(unhandled):
- shpimporter.INFO("Did not import values from fields: %s " % \
+ logger.info("Did not import values from fields: %s " % \
" ".join(unhandled))
try:
@@ -231,6 +234,6 @@
return geomType
destLayer.CommitTransaction()
except e:
- shpimporter.ERROR("Exception while committing transaction.")
+ logger.error("Exception while committing transaction.")
return geomType
diff -r eb2d6609387c -r 769593a84606 flys-backend/contrib/shpimporter/shpimporter.py
--- a/flys-backend/contrib/shpimporter/shpimporter.py Fri Feb 15 16:19:52 2013 +0100
+++ b/flys-backend/contrib/shpimporter/shpimporter.py Fri Feb 15 16:22:13 2013 +0100
@@ -6,6 +6,7 @@
import utils, optparse
import sys
import os
+import logging
from uesg import UESG
from axis import Axis
@@ -20,25 +21,16 @@
from catchments import Catchment
from dgm import insertRiverDgm
+logger = logging.getLogger("shpimporter")
-VERBOSE_DEBUG=2
-VERBOSE_INFO=1
-
-
-def DEBUG(msg):
- config = getConfig()
- if config.verbose >= VERBOSE_DEBUG:
- print "DEBUG: %s" % msg
-
-def INFO(msg):
- config = getConfig()
- if config.verbose >= VERBOSE_INFO:
- print "INFO: %s" % msg
-
-def ERROR(msg):
- config = getConfig()
- print "ERROR: %s" % msg
-
+def initialize_logging(level):
+ """Initializes the logging system"""
+ root = logging.getLogger()
+ root.setLevel(level)
+ hdlr = logging.StreamHandler()
+ fmt = logging.Formatter("%(levelname)s %(name)s: %(message)s")
+ hdlr.setFormatter(fmt)
+ root.addHandler(hdlr)
def getImporters(river_id, dbconn, dry_run):
return [
@@ -84,18 +76,25 @@
parser.add_option("--skip_dgm", type="int")
(config, args) = parser.parse_args()
+ if config.verbose > 1:
+ initialize_logging(logging.DEBUG)
+ elif config.verbose == 1:
+ initialize_logging(logging.INFO)
+ else:
+ initialize_logging(logging.WARN)
+
if config.directory == None:
- ERROR("No river directory specified!")
+ logger.error("No river directory specified!")
raise Exception("Invalid config")
if not config.ogr_connection:
if not config.host:
- ERROR("No database host specified!")
+ logger.error("No database host specified!")
raise Exception("Invalid config")
if not config.user:
- ERROR("No databaser user specified!")
+ logger.error("No databaser user specified!")
raise Exception("Invalid config")
if not config.password:
- ERROR("No password specified!")
+ logger.error("No password specified!")
raise Exception("Invalid config")
return config
@@ -139,11 +138,11 @@
return -1
if config == None:
- ERROR("Unable to read config from command line!")
+ logger.error("Unable to read config from command line!")
return
if config.dry_run > 0:
- INFO("You enable 'dry_run'. No database transaction will take place!")
+ logger.info("You enable 'dry_run'. No database transaction will take place!")
if config.ogr_connection:
connstr = config.ogr_connection
@@ -155,7 +154,7 @@
import cx_Oracle as dbapi
raw_connstr=connstr.replace("OCI:", "")
except ImportErrror:
- ERROR("Module cx_Oracle not found in: %s\n"
+ logger.error("Module cx_Oracle not found in: %s\n"
"Neccessary to connect to a Oracle Database.\n"
"Please refer to the installation "
"documentation." % sys.path)
@@ -165,7 +164,7 @@
import psycopg2 as dbapi
raw_connstr=connstr.replace("PG:", "")
except ImportError:
- ERROR("Module psycopg2 not found in: %s\n"
+ logger.error("Module psycopg2 not found in: %s\n"
"Neccessary to connect to a Posgresql Database.\n"
"Please refer to the installation "
"documentation." % sys.path)
@@ -174,7 +173,7 @@
dbconn = ogr.Open(connstr)
if dbconn == None:
- ERROR("Could not connect to database %s" % connstr)
+ logger.error("Could not connect to database %s" % connstr)
return -1
types = {}
@@ -197,21 +196,21 @@
river_id = utils.getRiverId(dbconn_raw, river_name)
if not river_id:
- INFO("Could not find river in database. Skipping: %s"
+ logger.info("Could not find river in database. Skipping: %s"
% river_name)
continue
else:
- INFO("Importing River: %s" % river_name)
+ logger.info("Importing River: %s" % river_name)
for importer in getImporters(river_id, dbconn, config.dry_run):
if skip_importer(config, importer):
- INFO("Skip import of '%s'" % importer.getName())
+ logger.info("Skip import of '%s'" % importer.getName())
continue
- INFO("Start import of '%s'" % importer.getName())
+ logger.info("Start import of '%s'" % importer.getName())
shapes = utils.findShapefiles(importer.getPath(config.directory))
- DEBUG("Found %i Shapefiles" % len(shapes))
+ logger.debug("Found %i Shapefiles" % len(shapes))
for shpTuple in shapes:
geomType = importer.walkOverShapes(shpTuple)
@@ -223,17 +222,17 @@
types[geomType] = 1
for key in types:
- DEBUG("%i x geometry type %s" % (types[key], key))
+ logger.debug("%i x geometry type %s" % (types[key], key))
if not config.skip_dgm:
dgmfilename = os.path.join(
config.directory, "..", "DGMs.csv")
if not os.access(dgmfilename, os.R_OK) or not \
os.path.isfile(dgmfilename):
- INFO("Could not find or access DGM file: %s \n"
+ logger.info("Could not find or access DGM file: %s \n"
"Skipping DGM import." % dgmfilename)
else:
- INFO("Inserting DGM meta information in 'dem' table.")
+ logger.info("Inserting DGM meta information in 'dem' table.")
insertRiverDgm(dbconn_raw, dgmfilename, river_name, config.dry_run)
if __name__ == '__main__':
diff -r eb2d6609387c -r 769593a84606 flys-backend/contrib/shpimporter/utils.py
--- a/flys-backend/contrib/shpimporter/utils.py Fri Feb 15 16:19:52 2013 +0100
+++ b/flys-backend/contrib/shpimporter/utils.py Fri Feb 15 16:22:13 2013 +0100
@@ -1,11 +1,14 @@
import os
import sys
-from shpimporter import DEBUG, INFO, ERROR
+import logging
+
try:
from osgeo import ogr
except ImportErrror:
import ogr
+logger = logging.getLogger("utils")
+
SHP='.shp'
SQL_SELECT_RIVER_ID="SELECT id FROM rivers WHERE name = %s"
@@ -16,7 +19,7 @@
if len(files) == 0:
continue
- DEBUG("Processing directory '%s' with %i files " % (root, len(files)))
+ logger.debug("Processing directory '%s' with %i files " % (root, len(files)))
for f in files:
idx = f.find(SHP)
More information about the Dive4elements-commits
mailing list