[Osaas-commits] r63 - in trunk: . server/osaas server/test

[scm-commit@wald.intevation.org]

Author: bh
Date: 2010-01-14 14:48:51 +0100 (Thu, 14 Jan 2010)
New Revision: 63

Modified:
   trunk/ChangeLog
   trunk/server/osaas/dbbackend.py
   trunk/server/test/test_dbbackend.py
Log:
Make error handling in the db backend more robust.  This should
fix the main problem of geospatial/issue30


Modified: trunk/ChangeLog
===================================================================
--- trunk/ChangeLog	2010-01-14 10:54:00 UTC (rev 62)
+++ trunk/ChangeLog	2010-01-14 13:48:51 UTC (rev 63)
@@ -1,5 +1,32 @@
 2010-01-14  Bernhard Herzog  <bh at intevation.de>
 
+	Make error handling in the db backend more robust.  This should
+	fix the main problem of geospatial/issue30
+
+	* server/osaas/dbbackend.py (DBBackend.log_exception): New method
+	to log exceptions analogous to the other log_* methods.
+	(DBBackend.ensure_connection): Add parameter force_new_connection
+	that can be used to force opening a new database connection
+	instead of reusing the existing one
+	(DBBackend.handle_record)
+	(DBBackend.insert_record_with_reconnect)
+	(DBBackend.insert_record): Split handle_record into handle_record,
+	insert_record_with_reconnect and insert_record.  handle_record
+	itself is the main entry point for the database insertion thread
+	and does the high level error handling, i.e. logging the record
+	contents if insertion fails for whatever reason.  insert_record
+	does the actual insertion and insert_record_with_reconnect calls
+	insert_record but retries if the database connection has been
+	closed unexpectedly (e.g. by a database server problem).  It also
+	forces new connections after each exception raised during the
+	insertion to make sure the connection object is in a usable state
+	for the next record.
+
+	* server/test/test_dbbackend.py (MockDbApi.OperationalError): Add
+	this class because it's used now by the dbbackend module.
+
+2010-01-14  Bernhard Herzog  <bh at intevation.de>
+
 	* server/osaas/dbbackend.py (DBBackend.get_insert_statement): New
 	method to generate and cache the insert statement for records.
 	(DBBackend.handle_record): Use the new method to get the insert

Modified: trunk/server/osaas/dbbackend.py
===================================================================
--- trunk/server/osaas/dbbackend.py	2010-01-14 10:54:00 UTC (rev 62)
+++ trunk/server/osaas/dbbackend.py	2010-01-14 13:48:51 UTC (rev 63)
@@ -78,7 +78,18 @@
         if self.logger is not None:
             self.logger.error(message, *args)
 
-    def ensure_connection(self):
+    def log_exception(self, message, *args):
+        if self.logger is not None:
+            self.logger.exception(message, *args)
+
+    def ensure_connection(self, force_new_connection=False):
+        if force_new_connection and self.connection is not None:
+            try:
+                self.connection.close()
+            except:
+                self.log_exception("Exception during connection.close()"
+                                   " while forcing a new connection")
+            self.connection = None
         if self.connection is None:
             self.log_debug("Connecting to database")
             self.connection = self.connect(*self.connection_parameters)
@@ -111,6 +122,39 @@
         return self.insert_statement
 
     def handle_record(self, record):
+        """Write record into the database.  Log the record if unsuccessful."""
+        try:
+            self.insert_record_with_reconnect(record)
+        except:
+            self.log_error("could not write record %s", record.raw_record)
+            raise
+
+    def insert_record_with_reconnect(self, record):
+        """Insert record into the database, reconnecting on some errors"""
+        try:
+            try:
+                self.insert_record(record)
+            except self.db_api.OperationalError:
+                self.log_exception("Got OperationalError."
+                                   " Trying again with a new connection"
+                                   " in case it's because of a DB server"
+                                   " connection problem")
+                self.ensure_connection(force_new_connection=True)
+                self.insert_record(record)
+        except:
+            # make sure to always use a new connection after an error
+            # happened.  If the network connection to the database
+            # vanishes unexpectedly, the DB API implementation might be
+            # left in an unusable state.  Also, we cannot just do this
+            # when an OperationalError is raised because some DB API
+            # implementations may raise different exceptions and even
+            # conforming implementations may raise other errors if the
+            # implementation
+            self.connection = None
+            raise
+
+    def insert_record(self, record):
+        """Insert record into the database."""
         statement = self.get_insert_statement()
         values = {}
         for f in self.rules[0].fields:
@@ -121,7 +165,6 @@
         try:
             cursor.execute(statement, values)
         except:
-            self.log_error("could not write record %s", record.raw_record)
             self.connection.rollback()
             raise
         else:

Modified: trunk/server/test/test_dbbackend.py
===================================================================
--- trunk/server/test/test_dbbackend.py	2010-01-14 10:54:00 UTC (rev 62)
+++ trunk/server/test/test_dbbackend.py	2010-01-14 13:48:51 UTC (rev 63)
@@ -1,4 +1,4 @@
-# Copyright (C) 2007, 2008 by Intevation GmbH
+# Copyright (C) 2007, 2008, 2010 by Intevation GmbH
 # Authors:
 # Bernhard Herzog <bh at intevation.de>
 #
@@ -55,7 +55,10 @@
 
     paramstyle = "named"
 
+    class OperationalError(Exception):
+        pass
 
+
 class MockLogger(object):
 
     def __init__(self):