[Dive4elements-commits] [PATCH 1 of 2] Add handling of River Names and remove target_src parameter
Wald Commits
scm-commit at wald.intevation.org
Wed Feb 13 12:07:22 CET 2013
# HG changeset patch
# User Andre Heinecke <aheinecke at intevation.de>
# Date 1360753350 -3600
# Branch dami
# Node ID 174fbaa3d344e87b635656dcab4e3fe385672d38
# Parent 1469066cc7d92783c5baed57295ba82462711b7f
Add handling of River Names and remove target_src parameter
This is the first step to make the shpimporter into a more
generic geo importer that communicates directly (not only over
ogr) with the database.
Untested with Oracle
diff -r 1469066cc7d9 -r 174fbaa3d344 flys-backend/contrib/shpimporter/importer.py
--- a/flys-backend/contrib/shpimporter/importer.py Mon Feb 04 17:54:39 2013 +0100
+++ b/flys-backend/contrib/shpimporter/importer.py Wed Feb 13 12:02:30 2013 +0100
@@ -9,12 +9,12 @@
class Importer:
- def __init__(self, config, dbconn):
- self.config = config
+ def __init__(self, river_id, dbconn, dry_run):
+ self.river_id = river_id
self.dbconn = dbconn
- self.river_id = config.river_id
+ self.dry_run = dry_run
self.dest_srs = osr.SpatialReference()
- self.dest_srs.ImportFromEPSG(config.target_srs)
+ self.dest_srs.ImportFromEPSG(31467)
self.handled_fields = []
self.tracking_import = False
self.srcLayer = None
@@ -219,7 +219,7 @@
" ".join(unhandled))
try:
- if self.config.dry_run > 0:
+ if self.dry_run:
return geomType
destLayer.CommitTransaction()
except e:
diff -r 1469066cc7d9 -r 174fbaa3d344 flys-backend/contrib/shpimporter/shpimporter.py
--- a/flys-backend/contrib/shpimporter/shpimporter.py Mon Feb 04 17:54:39 2013 +0100
+++ b/flys-backend/contrib/shpimporter/shpimporter.py Wed Feb 13 12:02:30 2013 +0100
@@ -4,6 +4,8 @@
import ogr
import utils, optparse
+import sys
+import os
from uesg import UESG
from axis import Axis
@@ -38,22 +40,22 @@
print "ERROR: %s" % msg
-def getImporters(config, dbconn):
+def getImporters(river_id, dbconn, dry_run):
return [
- Axis(config, dbconn),
- KM(config, dbconn),
- CrosssectionTrack(config, dbconn),
- Line(config, dbconn),
- Fixpoint(config, dbconn),
- Building(config, dbconn),
- Floodplain(config, dbconn),
- HydrBoundary(config, dbconn),
- HydrBoundaryPoly(config, dbconn),
- HWSLines(config, dbconn),
- HWSPoints(config, dbconn),
- GaugeLocation(config, dbconn),
- Catchment(config, dbconn),
- UESG(config, dbconn)
+ Axis(river_id, dbconn, dry_run),
+ KM(river_id, dbconn, dry_run),
+ CrosssectionTrack(river_id, dbconn, dry_run),
+ Line(river_id, dbconn, dry_run),
+ Fixpoint(river_id, dbconn, dry_run),
+ Building(river_id, dbconn, dry_run),
+ Floodplain(river_id, dbconn, dry_run),
+ HydrBoundary(river_id, dbconn, dry_run),
+ HydrBoundaryPoly(river_id, dbconn, dry_run),
+ HWSLines(river_id, dbconn, dry_run),
+ HWSPoints(river_id, dbconn, dry_run),
+ GaugeLocation(river_id, dbconn, dry_run),
+ Catchment(river_id, dbconn, dry_run),
+ UESG(river_id, dbconn, dry_run)
]
@@ -64,7 +66,7 @@
parser.add_option("--host", type="string")
parser.add_option("--user", type="string")
parser.add_option("--password", type="string")
- parser.add_option("--river_id", type="int")
+ parser.add_option("--river_name", type="string")
parser.add_option("--verbose", type="int", default=1)
parser.add_option("--dry_run", type="int", default=0)
parser.add_option("--ogr_connection", type="string")
@@ -96,9 +98,6 @@
if not config.password:
ERROR("No password specified!")
raise Exception("Invalid config")
- if config.river_id == None:
- ERROR("No river id specified!")
- raise Exception("Invalid config")
return config
@@ -135,7 +134,6 @@
return False
-
def main():
config=None
try:
@@ -155,33 +153,75 @@
else:
connstr = 'OCI:%s/%s@%s' % (config.user, config.password, config.host)
+ if 'OCI:' in connstr:
+ try:
+ import cx_Oracle as dbapi
+ raw_connstr=connstr.replace("OCI:", "")
+ except ImportErrror:
+ ERROR("Module cx_Oracle not found in: %s\n"
+ "Neccessary to connect to a Oracle Database.\n"
+ "Please refer to the installation "
+ "documentation." % sys.path)
+
+ else: # Currently only support for oracle and postgres
+ try:
+ import psycopg2 as dbapi
+ raw_connstr=connstr.replace("PG:", "")
+ except ImportErrror:
+ ERROR("Module psycopg2 not found in: %s\n"
+ "Neccessary to connect to a Posgresql Database.\n"
+ "Please refer to the installation "
+ "documentation." % sys.path)
+
+ dbconn_raw = dbapi.connect(raw_connstr)
dbconn = ogr.Open(connstr)
if dbconn == None:
ERROR("Could not connect to database %s" % connstr)
return -1
- importers = getImporters(config, dbconn)
types = {}
- for importer in importers:
- if skip_importer(config, importer):
- INFO("Skip import of '%s'" % importer.getName())
+ directories = []
+ if not config.river_name:
+ for file in os.listdir(config.directory):
+ if os.path.isdir(file):
+ directories.append(os.path.join(file))
+ else:
+ directories.append(config.directory)
+
+ for directory in directories:
+ if not config.river_name:
+ river_name = os.path.basename(os.path.normpath(directory))
+ else:
+ river_name = config.river_name
+ river_id = utils.getRiverId(dbconn_raw, river_name)
+
+ if not river_id:
+ INFO("Could not find river in database. Skipping: %s"
+ % river_name)
continue
+ else:
+ INFO("Importing River: %s" % river_name)
- INFO("Start import of '%s'" % importer.getName())
+ for importer in getImporters(river_id, dbconn, config.dry_run):
+ if skip_importer(config, importer):
+ INFO("Skip import of '%s'" % importer.getName())
+ continue
- shapes = utils.findShapefiles(importer.getPath(config.directory))
- DEBUG("Found %i Shapefiles" % len(shapes))
+ INFO("Start import of '%s'" % importer.getName())
- for shpTuple in shapes:
- geomType = importer.walkOverShapes(shpTuple)
- try:
- if geomType is not None:
- num = types[geomType]
- types[geomType] = num+1
- except:
- types[geomType] = 1
+ shapes = utils.findShapefiles(importer.getPath(config.directory))
+ DEBUG("Found %i Shapefiles" % len(shapes))
+
+ for shpTuple in shapes:
+ geomType = importer.walkOverShapes(shpTuple)
+ try:
+ if geomType is not None:
+ num = types[geomType]
+ types[geomType] = num+1
+ except:
+ types[geomType] = 1
for key in types:
DEBUG("%i x geometry type %s" % (types[key], key))
diff -r 1469066cc7d9 -r 174fbaa3d344 flys-backend/contrib/shpimporter/utils.py
--- a/flys-backend/contrib/shpimporter/utils.py Mon Feb 04 17:54:39 2013 +0100
+++ b/flys-backend/contrib/shpimporter/utils.py Wed Feb 13 12:02:30 2013 +0100
@@ -1,8 +1,13 @@
import os
import sys
from shpimporter import DEBUG, INFO, ERROR
+try:
+ from osgeo import ogr
+except ImportErrror:
+ import ogr
SHP='.shp'
+SQL_SELECT_ID="SELECT id FROM %s WHERE %s = '%s'"
def findShapefiles(path):
shapes = []
@@ -20,6 +25,20 @@
return shapes
+def getRiverId(dbconn, name):
+ """
+ Returns the id of the river "name"
+ Dbconn must be a python database connection api compliant object
+ """
+ cur = dbconn.cursor()
+ select_stmt = SQL_SELECT_ID % ("rivers", "name", name)
+ cur.execute(select_stmt)
+ row = cur.fetchone()
+ if row:
+ return row[0]
+ else:
+ return 0
+
def getUTF8(string):
"""
Tries to convert the string to a UTF-8 encoding by first checking if it
More information about the Dive4elements-commits
mailing list