[Osaas-commits] r77 - in trunk: . contrib contrib/billing
scm-commit@wald.intevation.org
scm-commit at wald.intevation.org
Mon Oct 4 10:13:00 CEST 2010
Author: iweinzierl
Date: 2010-10-04 10:12:58 +0200 (Mon, 04 Oct 2010)
New Revision: 77
Added:
trunk/contrib/billing/
trunk/contrib/billing/billing.cfg
trunk/contrib/billing/billing.py
trunk/contrib/billing/billing.sql
trunk/contrib/billing/wfs_requests.sql
Modified:
trunk/ChangeLog
Log:
Added a python script (and relevant sql scripts) to update the number of features of GetFeature requests stored in the database table where osaas is logging to.
Modified: trunk/ChangeLog
===================================================================
--- trunk/ChangeLog 2010-03-22 15:15:46 UTC (rev 76)
+++ trunk/ChangeLog 2010-10-04 08:12:58 UTC (rev 77)
@@ -1,3 +1,18 @@
+2010-10-04 Ingo Weinzierl <ingo.weinzierl at intevation.de>
+
+ * contrib/billing/billing.py,
+ contrib/billing/billing.cfg: A python script and an example configuration.
+ The script looks for GetFeature requests stored in a database table and
+ updates the number of features returned by the specific request of the
+ entries.
+
+ * contrib/billing/wfs_requests.sql: An sql script that creates a table
+ that stores wfs requests. OSAAS is logging to such a table.
+
+ * contrib/billing/billing.sql: An additional script for the
+ wfs_requests.sql script - it adds a new column 'billtime' that saves
+ the time when the billing.py script has been executed.
+
2010-03-22 Stephan Holl <stephan.holl at intevation.de>
* packaging/rpm/osaas-suse10.3-gp.spec: Updated to reflect the
Added: trunk/contrib/billing/billing.cfg
===================================================================
--- trunk/contrib/billing/billing.cfg 2010-03-22 15:15:46 UTC (rev 76)
+++ trunk/contrib/billing/billing.cfg 2010-10-04 08:12:58 UTC (rev 77)
@@ -0,0 +1,35 @@
+# Demo configuration for the billing script.
+#
+# This file defines some basic configuration settings.
+#
+
+[database]
+## !!! This section is necessary for running this script. At least a database
+## !!! and a user have to be defined, otherwise you are not able to run this
+## !!! script.
+
+## The host where the database is running on. If this option is not used,
+## 'localhost' is used as hostname.
+#host=localhost
+
+## The database where osaas is logging to.
+database=osaas_logging
+
+## A user that is allowed to access the database specified above.
+user=postgres
+
+## The password that is required to connect to database.
+#password=
+
+[logging]
+## Uncomment the following line to specify the logfile used for internal
+## logging. By default, a file named 'billing.log' is used for logging.
+logfile=billing.log
+
+## The following line defines the loglevel for the script. Currently, there are
+## three loglevels defined:
+## 1: the script just logs failures the occured while processing.
+## 2: the script shows failures and some additional information.
+## 3: the script is very verbose - shows failures, additional information and a
+## lot of information about the process itself (recommended for developers).
+loglevel=2
Added: trunk/contrib/billing/billing.py
===================================================================
--- trunk/contrib/billing/billing.py 2010-03-22 15:15:46 UTC (rev 76)
+++ trunk/contrib/billing/billing.py 2010-10-04 08:12:58 UTC (rev 77)
@@ -0,0 +1,360 @@
+#! /usr/bin/env python
+#
+# Copyright (C) 2010 by Intevation GmbH
+# Authors:
+# Ingo Weinzierl <ingo.weinzierl at intevation.de>
+#
+# This program is free software under the LGPL (>=v2.1)
+# Read the file LGPL.txt coming with the software for details.
+#
+
+import logging
+import psycopg2 as db
+import sys
+import urllib
+
+from ConfigParser import SafeConfigParser
+from optparse import OptionParser
+from lxml import etree
+from StringIO import StringIO
+from urlparse import urlparse
+
+DEFAULT_CONFIG = "billing.cfg"
+DEFAULT_LOGFILE = "billing.log"
+
+REQUEST_POST = 1
+REQUEST_GET = 2
+
+NS_WFS = "http://www.opengis.net/wfs"
+
+SQL_SELECT_OPEN_BILLINGS = """SELECT id, wfsidintern, wfsidextern,
+requestformat, rawrequest AS TEXT, numfeatures, billtime FROM requests
+WHERE billtime IS NULL"""
+SQL_UPDATE_NUM_FEATURES = """UPDATE requests SET numfeatures = %(num)s,
+billtime = CURRENT_TIMESTAMP where id =
+%(id)s"""
+
+config=None
+
+def read_config(filename):
+ """Reads the configuration from the file given by filename."""
+ config = SafeConfigParser()
+ config.read([filename])
+ return config
+
+def get_config_option(section, option, default=None):
+ if not config:
+ print "No config given yet."
+ return default
+
+ if not config.has_option(section, option):
+ return default
+ return config.get(section, option)
+
+def setup_logging():
+ """Sets up the logging."""
+
+ loglevel = logging.WARN
+ level = get_config_option("logging", "loglevel", 1)
+ if level and int(level) >= 3:
+ print "Setup logging to DEBUG"
+ loglevel = logging.DEBUG
+ elif level and int(level) == 2:
+ print "Setup logging to INFO"
+ loglevel = logging.INFO
+
+ logfile = get_config_option("logging", "logfile", DEFAULT_LOGFILE)
+
+ file_handler = logging.FileHandler(logfile)
+ file_handler.setFormatter(logging.Formatter(
+ "%(asctime)s %(levelname)s %(message)s"))
+ file_handler.setLevel(loglevel)
+
+ console_handler = logging.StreamHandler(sys.stdout)
+ console_handler.setFormatter(logging.Formatter(
+ "--> %(asctime)s %(levelname)s %(message)s"))
+ console_handler.setLevel(logging.WARN)
+
+ global logger
+ logger = logging.getLogger(__name__)
+ logger.setLevel(logging.DEBUG)
+ logger.addHandler(console_handler)
+ logger.addHandler(file_handler)
+
+
+class Request(object):
+ """This class represents a very simple request object.
+
+ It stores some basic things like the server url, the request format
+ (1=HTTP-POST, 2=HTTP-GET), its data, the number of features that have been
+ retrieved by this request earlier and the time where the billing has taken
+ place."""
+ def __init__(self, id, wfsidintern, wfsidextern, requestformat,
+ rawrequest, numfeatures, cashtime):
+ self.id = id
+ self.wfsidintern = wfsidintern
+ self.wfsidextern = wfsidextern
+ self.requestformat = requestformat
+ self.rawrequest = rawrequest
+ self.numfeatures = numfeatures
+ self.cashtime = cashtime
+
+ def handle_response(self, doc):
+ """ This method handles the response object of the request. Currently,
+ it searches for an xml element with the xpath
+ '/FeatureCollection/@numberOfFeatures' where the number of features
+ retrieved by this request should be stored. The default namespace for
+ this xpath is NS_WFS. If a valid quantity is found by the xpath
+ expression, the number of features is saved to database."""
+ try:
+ logger.debug("Response finished.")
+ response = doc.xpath('/g:FeatureCollection/@numberOfFeatures',
+ namespaces={'g': NS_WFS})
+ if not response:
+ logger.warn("GetFeature request retrieved no number of "
+ "features.")
+ return False
+
+ numFeatures = int(response[0])
+ logger.info("GetFeature retrieved %d features." %
+ numFeatures)
+ self.numfeatures = numFeatures
+ return self.update_num_features()
+ except ValueError, e:
+ logger.error("The response did not retrieve a valid quantity.")
+ return False
+
+ def update_num_features(self):
+ """ This method opens a database connection and updates the request
+ object's number of features using the SQL query defined by
+ SQL_UPDATE_NUM_FEATURES. The return value of this function is the number
+ of features that have been written into database, or False if the update
+ process failed."""
+ db_conn = open_connection()
+
+ if not db_conn:
+ return False
+
+ try:
+ logger.debug("Execute update statement for request id %d" %
+ self.id)
+ cur = db_conn.cursor()
+ cur.execute(SQL_UPDATE_NUM_FEATURES, { 'id': self.id,
+ 'num': self.numfeatures})
+ db_conn.commit()
+ close_connection(cur, db_conn)
+ logger.info("Update for request id %d completed (%d features)"
+ % (self.id, self.numfeatures))
+ return self.numfeatures
+ except:
+ logger.error("Could not update request id %r" % self.id)
+ return False
+
+
+class GETRequest(Request):
+ """This request object represents HTTP-GET request objects and inherits from
+ Request."""
+
+ def is_get_feature_request(self):
+ """This method returns True, if this request object represents an OGC
+ WFS GetFeature request, otherwise it returns False."""
+ if 'request=getfeature' in str(self.rawrequest).lower():
+ return True
+ return False
+
+ def create_request_url(self):
+ """This method modifies the URL of this GetFeature request and returns
+ an URL which response is the number of features of the GetFeature
+ request represented by this object."""
+ req_url = None
+ if "?" in self.wfsidintern:
+ req_url = "%s&%s" % (self.wfsidintern, self.rawrequest)
+ else:
+ req_url = "%s?%s" % (self.wfsidintern, self.rawrequest)
+
+ req_url = urllib.unquote(req_url)
+ if 'resulttype' in req_url.lower():
+ url_parts = urlparse(req_url)
+ req_url = req_url.split("?")[0]
+ if url_parts and url_parts[4]:
+ params = url_parts[4].split("&") or []
+ for param in params:
+ if "?" in req_url:
+ req_url += "&"
+ else:
+ req_url += "?"
+
+ if 'resulttype' in param.lower():
+ param = "resultType=hits"
+ req_url += param
+ else:
+ req_url += '&resultType=hits'
+ logger.debug("request url = %s" % req_url)
+ return req_url
+
+ def bill(self):
+ """ This method should be called to take account for the request. If
+ this object represents a GetFeature request, a modified version of this
+ request is used to query the number of feature that would have been
+ retrieved by this GetFeature request. If the modified version of this
+ request returns a valid quantity, this number is used to update the row
+ in the database. If everything was ok in this process, this method
+ returns the number of features of this request, otherwise False."""
+ req_url = self.create_request_url()
+ try:
+ doc = etree.parse(req_url)
+ return self.handle_response(doc)
+ except IOError:
+ logger.error("Could not connect to server %s" %
+ self.wfsidintern)
+ except:
+ logger.error("Could not bill request with database id %r" %
+ self.id)
+ return False
+
+
+class POSTRequest(Request):
+ """This request object represents HTTP-POST request objects and inherits
+ from Request."""
+
+ def is_get_feature_request(self):
+ """This method returns True, if this request object represents an OGC
+ WFS GetFeature request, otherwise it returns False."""
+ doc = etree.parse(StringIO(self.rawrequest))
+ if doc and doc.xpath('/g:GetFeature', namespaces={'g': NS_WFS}):
+ return True
+ return False
+
+ def bill(self, db_conn=None):
+ """ This method should be called to take account for the request. If
+ this object represents a GetFeature request, a modified version of this
+ request's raw data is used to query the number of feature that would
+ have been retrieved by this GetFeature request. If the modified version
+ of this request returns a valid quantity, this number is used to update
+ the row in the database. If everything was ok in this process, this
+ method returns the number of features of this request, otherwise
+ False."""
+ doc = etree.parse(StringIO(self.rawrequest))
+ req = doc.xpath('/g:GetFeature', namespaces={'g': NS_WFS})
+
+ if not req or not req[0]:
+ logger.error("Could not find request parameters for request id"
+ "%r" % self.id)
+ return False
+
+ req[0].set('resultType', 'hits')
+ opener = urllib.URLopener()
+ data = etree.tostring(doc)
+ conn = opener.open(self.wfsidintern, data)
+
+ if not conn:
+ logger.error("Could not connect to host '%r'." %
+ self.wfsidintern)
+ return False
+
+ response = etree.parse(conn)
+ return self.handle_response(response)
+
+
+def open_connection():
+ host = get_config_option("database", "host", "localhost")
+ dbname = get_config_option("database", "database")
+ user = get_config_option("database", "user")
+ password = get_config_option("database", "password", "")
+
+ try:
+ conn = db.connect(host=host, database=dbname, user=user,
+ password=password)
+ return conn
+ except:
+ logger.critical("Could not connect to database (host='%s', database="
+ "'%s', user='%s'" % (host, dbname, user))
+
+
+def close_connection(*cur):
+ for c in cur:
+ if c:
+ try: c.close()
+ except: pass
+
+
+def load_requests():
+ requests = []
+ try:
+ conn = open_connection()
+ if not conn:
+ return None
+
+ cur = conn.cursor()
+ cur.execute(SQL_SELECT_OPEN_BILLINGS)
+ while True:
+ row = cur.fetchone()
+ if not row: break
+
+ req = None
+ if row[3] == REQUEST_POST:
+ req = POSTRequest(*row)
+ elif row[3] == REQUEST_GET:
+ req = GETRequest(*row)
+ else:
+ logger.warn("Unknown request format found: %r" % row[3])
+
+ if req and not req.cashtime and req.is_get_feature_request():
+ requests.append(req)
+ close_connection(cur, conn)
+ return requests
+ except:
+ logger.error("An error occured while connecting to database.")
+ return None
+
+
+def main():
+ parser = OptionParser()
+ parser.add_option("-c", "--config-file", default=DEFAULT_CONFIG)
+ opts, rest = parser.parse_args()
+
+ global config
+ config=read_config(opts.config_file)
+
+ setup_logging()
+ print "Start billing script."
+ print "===============================================\n"
+
+ try:
+ requests = load_requests()
+ if not requests:
+ return
+
+ logger.info("Found %d open billings for GetFeature requests." %
+ len(requests))
+
+ failed = 0
+ success = 0
+ for req in requests:
+ logger.info("================================")
+ logger.info("Process request with db id %d (format = %d)" % (
+ req.id, req.requestformat))
+ try:
+ if not req.bill():
+ failed += 1
+ else:
+ success += 1
+ except:
+ logger.error("Could not bill the request with id %d" % req.id)
+
+ print "\nScript ended successfully."
+ print "\n==============================================="
+ print "== Summary: Found %d open requests." % len(requests)
+ print "== Summary: Sucessfully billed %d requests." % success
+ print "== Summary: %d requests failed." % failed
+ print "== (See the logfile '%s' for more details)" % get_config_option(
+ "logging",
+ "logfile",
+ DEFAULT_LOGFILE)
+ except:
+ logger.error("An unexpected error occured - the script did not "
+ "finish regularly!")
+
+
+if __name__ == '__main__':
+ main()
Property changes on: trunk/contrib/billing/billing.py
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/contrib/billing/billing.sql
===================================================================
--- trunk/contrib/billing/billing.sql 2010-03-22 15:15:46 UTC (rev 76)
+++ trunk/contrib/billing/billing.sql 2010-10-04 08:12:58 UTC (rev 77)
@@ -0,0 +1 @@
+ALTER TABLE requests ADD COLUMN billtime TIMESTAMP WITH TIME ZONE;
Added: trunk/contrib/billing/wfs_requests.sql
===================================================================
--- trunk/contrib/billing/wfs_requests.sql 2010-03-22 15:15:46 UTC (rev 76)
+++ trunk/contrib/billing/wfs_requests.sql 2010-10-04 08:12:58 UTC (rev 77)
@@ -0,0 +1,19 @@
+CREATE SEQUENCE requests_id_seq
+ INCREMENT BY 1
+ NO MAXVALUE
+ NO MINVALUE
+ CACHE 1;
+
+CREATE TABLE requests (
+ id INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('requests_id_seq'::regclass),
+ wfsidintern CHARACTER VARYING(100),
+ wfsidextern CHARACTER VARYING(100),
+ username CHARACTER VARYING(20),
+ starttime TIMESTAMP WITH TIME ZONE,
+ endtime TIMESTAMP WITH TIME ZONE,
+ requestformat INTEGER,
+ rawrequest BYTEA,
+ numfeatures INTEGER
+);
+
+ALTER TABLE public.requests OWNER TO osaas;
More information about the Osaas-commits
mailing list