[PATCH 07 of 12] Importer (s/u-info) extensions: flow-depth: uniform formatting of from-to series names,
Wald Commits
scm-commit at wald.intevation.org
Mon Mar 23 16:38:43 CET 2020
# HG changeset patch
# User mschaefer
# Date 1584973299 -3600
# Mon Mar 23 15:21:39 2020 +0100
# Node ID 31549fdfaf4f3e74dafdb0c7f102a27b5c0b0015
# Parent 1f57381b3bb579913a3742216899abe83fc888c7
Importer (s/u-info) extensions: flow-depth: uniform formatting of from-to series names,
warning instead of cancelling in case of missing column values,
detecting, logging and skipping columns with wrong unit,
better counting of inserted/updated values for each column
diff -r 1f57381b3bb5 -r 31549fdfaf4f backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthColumnSeriesImport.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthColumnSeriesImport.java Mon Mar 23 15:16:35 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthColumnSeriesImport.java Mon Mar 23 15:21:39 2020 +0100
@@ -18,7 +18,7 @@
import org.dive4elements.river.model.River;
import org.dive4elements.river.model.sinfo.FlowDepthColumn;
import org.dive4elements.river.model.sinfo.FlowDepthValue;
-import org.hibernate.SQLQuery;
+import org.hibernate.Query;
import org.hibernate.Session;
/**
@@ -55,7 +55,11 @@
public FlowDepthColumnSeriesImport(final String filename, final FlowDepthSeriesImport parent, final String colName, final File relativeFilePath) {
super(filename);
this.parent = parent;
- this.colName = colName;
+ final String[] items = (colName == null) ? new String[] {} : colName.split("\\-");
+ if (items.length == 2)
+ this.colName = items[0].trim() + " - " + items[1].trim();
+ else
+ this.colName = colName;
this.relativeFilePath = relativeFilePath;
}
@@ -91,20 +95,22 @@
return log;
}
- @Override
- public List<FlowDepthColumn> querySeriesItem(final Session session, final River river) {
- /*
- * final Query query = session.createQuery("FROM FlowDepthColumn WHERE (FlowDepth=:parent) AND lower(name)=:colname");
- * query.setParameter("parent", this.parent.getPeer(river));
- * query.setParameter("colname", this.colName.toLowerCase());
- */
- // FIXME the normal query raises a null pointer exception
- final SQLQuery query = session.createSQLQuery("SELECT * FROM flow_depth_column WHERE (flow_depth_id=:parent) AND (lower(name)=:colname)");
- query.setParameter("parent", this.parent.getPeer(river).getId());
+ private List<FlowDepthColumn> querySeriesItem(final Session session, final River river) {
+ final Query query = session.createQuery("FROM FlowDepthColumn WHERE flowDepth=:parent AND lower(name)=:colname");
+ query.setParameter("parent", this.parent.getPeer(river));
query.setParameter("colname", this.colName.toLowerCase());
return query.list();
}
+ @Override
+ public List<FlowDepthColumn> querySeriesItem(final Session session, final River river, final boolean doQueryParent) {
+ final Query query = session.createQuery("FROM FlowDepthColumn c INNER JOIN c.flowDepth s"
+ + " WHERE s.river=:river AND lower(s.filename)=:filename AND lower(c.name)=:colname");
+ query.setParameter("river", river);
+ query.setParameter("filename", this.filename.toLowerCase());
+ query.setParameter("colname", this.colName.toLowerCase());
+ return query.list();
+ }
@Override
public FlowDepthColumn createSeriesItem(final River river) {
diff -r 1f57381b3bb5 -r 31549fdfaf4f backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthKmLineImport.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthKmLineImport.java Mon Mar 23 15:16:35 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthKmLineImport.java Mon Mar 23 15:21:39 2020 +0100
@@ -46,7 +46,7 @@
@Override
protected FlowDepthValue queryValueItem(final Session session, final FlowDepthColumn parent) {
- final Query query = session.createQuery("FROM FlowDepthValue WHERE (FlowDepthColumn=:parent)"
+ final Query query = session.createQuery("FROM FlowDepthValue WHERE (flowDepthColumn=:parent)"
+ " AND (station BETWEEN (:station-0.0001) AND (:station+0.0001))");
query.setParameter("parent", parent);
query.setParameter("station", this.station);
diff -r 1f57381b3bb5 -r 31549fdfaf4f backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthSeriesImport.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthSeriesImport.java Mon Mar 23 15:16:35 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/FlowDepthSeriesImport.java Mon Mar 23 15:21:39 2020 +0100
@@ -132,7 +132,7 @@
return this.peer;
}
- private List<FlowDepth> querySeriesItem(final Session session, final River river) {
+ public List<FlowDepth> querySeriesItem(final Session session, final River river) {
final Query query = session.createQuery("FROM FlowDepth WHERE river=:river AND lower(filename)=:filename");
query.setParameter("river", river);
query.setParameter("filename", this.filename.toLowerCase());
diff -r 1f57381b3bb5 -r 31549fdfaf4f backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthColumnParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthColumnParser.java Mon Mar 23 15:16:35 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthColumnParser.java Mon Mar 23 15:21:39 2020 +0100
@@ -44,8 +44,7 @@
/***** CONSTRUCTORS *****/
public FlowDepthColumnParser(final File importPath, final File rootRelativePath, final ImportRiver river, final FlowDepthSeriesImport parent,
- final int colIndex,
- final String colName) {
+ final int colIndex, final String colName) {
super(importPath, rootRelativePath, river);
this.parent = parent;
this.colIndex = colIndex;
@@ -62,7 +61,7 @@
@Override
protected void logStartInfo() {
- getLog().info(String.format("Start parsing column %d '%s':;'%s'", this.colIndex, this.colName, this.rootRelativePath));
+ getLog().info(String.format("Start parsing column %d '%s':;'%s'", this.colIndex + 1, this.colName, this.rootRelativePath));
}
@Override
@@ -78,7 +77,11 @@
@Override
protected FlowDepthKmLineImport createKmLineImport(final Double km, final String[] values) {
- final double tkheight = parseDoubleWithNull(values[this.colIndex]).doubleValue();
- return new FlowDepthKmLineImport(km, tkheight);
+ final Number value = parseDoubleCheckNull(values, this.colIndex);
+ if ((value == null) || Double.isNaN(value.doubleValue())) {
+ logLineWarning("Column %d: " + INVALID_VALUE_ERROR_FORMAT, this.colIndex + 1, "depth");
+ return null;
+ }
+ return new FlowDepthKmLineImport(km, value.doubleValue());
}
}
diff -r 1f57381b3bb5 -r 31549fdfaf4f backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthParser.java Mon Mar 23 15:16:35 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthParser.java Mon Mar 23 15:21:39 2020 +0100
@@ -12,7 +12,6 @@
import java.io.File;
import java.io.FileInputStream;
-import java.io.IOException;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.util.ArrayList;
@@ -23,13 +22,16 @@
import org.apache.log4j.Logger;
import org.dive4elements.river.importer.Config;
import org.dive4elements.river.importer.ImportRiver;
+import org.dive4elements.river.importer.ImporterSession;
import org.dive4elements.river.importer.common.AbstractParser;
import org.dive4elements.river.importer.common.ParsingState;
import org.dive4elements.river.importer.sinfo.importitem.FlowDepthColumnSeriesImport;
import org.dive4elements.river.importer.sinfo.importitem.FlowDepthKmLineImport;
import org.dive4elements.river.importer.sinfo.importitem.FlowDepthSeriesImport;
+import org.dive4elements.river.model.sinfo.FlowDepth;
import org.dive4elements.river.model.sinfo.FlowDepthColumn;
import org.dive4elements.river.model.sinfo.FlowDepthValue;
+import org.hibernate.Session;
/**
* Reads and parses the header of a flow depth file and handles the parse and store of the columns
@@ -53,7 +55,7 @@
private static final Pattern COLUMN_TITLE = Pattern.compile("Flie((.)|(ss))tiefe\\s*\\((.+?)\\)\\s*\\[m\\].*", Pattern.CASE_INSENSITIVE);
- private final FlowDepthSeriesImport tkhGroup;
+ private final FlowDepthSeriesImport flowdepthGroup;
private final List<FlowDepthColumnParser> colParsers;
@@ -62,8 +64,8 @@
public FlowDepthParser(final File importPath, final File rootRelativePath, final ImportRiver river) {
super(importPath, rootRelativePath, river);
- this.tkhGroup = new FlowDepthSeriesImport(importPath.getName().replaceAll("\\.csv", ""));
- this.seriesHeader = new FlowDepthColumnSeriesImport(this.tkhGroup.getFilename(), this.tkhGroup, null, null);
+ this.flowdepthGroup = new FlowDepthSeriesImport(importPath.getName().replaceAll("\\.csv", ""));
+ this.seriesHeader = new FlowDepthColumnSeriesImport(this.flowdepthGroup.getFilename(), this.flowdepthGroup, null, null);
this.colParsers = new ArrayList<>();
}
@@ -94,7 +96,7 @@
}
@Override
- public void parse() throws IOException {
+ public void parse() throws Exception {
getLog().info("Start parsing:;'" + this.rootRelativePath + "'");
// this.seriesHeader = createSeriesImport(this.importPath.getName().replaceAll("\\.csv", ""));
this.metaPatternsMatched.clear();
@@ -106,7 +108,7 @@
this.in = new LineNumberReader(new InputStreamReader(new FileInputStream(this.importPath), ENCODING));
}
catch (final Exception e) {
- logError("Could not open (" + e.getMessage() + ")");
+ logError("Could not open (%s)", e.getMessage());
this.headerParsingState = ParsingState.STOP;
}
this.currentLine = null;
@@ -118,6 +120,8 @@
if (this.currentLine.isEmpty())
continue;
handleMetaLine();
+ if (this.headerParsingState == ParsingState.DONE)
+ checkMetaData();
}
}
finally {
@@ -152,7 +156,7 @@
final Matcher m = META_YEAR.matcher(this.currentLine);
if (m.matches()) {
this.metaPatternsMatched.add(META_YEAR);
- this.tkhGroup.setYear(Integer.parseInt(m.group(1)));
+ this.flowdepthGroup.setYear(Integer.parseInt(m.group(1)));
return true;
}
return false;
@@ -167,7 +171,7 @@
final Matcher m = META_SOUNDING.matcher(this.currentLine);
if (m.matches()) {
this.metaPatternsMatched.add(META_SOUNDING);
- this.tkhGroup.setSounding_info(parseMetaInfo(m.group(1).trim()));
+ this.flowdepthGroup.setSounding_info(parseMetaInfo(m.group(1).trim()));
return true;
}
return false;
@@ -177,7 +181,7 @@
final Matcher m = META_EVALUATOR.matcher(this.currentLine);
if (m.matches()) {
this.metaPatternsMatched.add(META_EVALUATOR);
- this.tkhGroup.setEvaluation_by(parseMetaInfo(m.group(1).trim()));
+ this.flowdepthGroup.setEvaluation_by(parseMetaInfo(m.group(1).trim()));
return true;
}
return false;
@@ -187,27 +191,40 @@
protected boolean handleMetaColumnTitles() {
if (!super.handleMetaColumnTitles())
return false;
- this.tkhGroup.setKmrange_info(this.seriesHeader.getKmrange_info());
- this.tkhGroup.setNotes(this.seriesHeader.getNotes());
+ this.flowdepthGroup.setKmrange_info(this.seriesHeader.getKmrange_info());
+ this.flowdepthGroup.setNotes(this.seriesHeader.getNotes());
for (int i = 1; i <= this.columnTitles.size() - 1; i++) {
final Matcher m = COLUMN_TITLE.matcher(this.columnTitles.get(i));
if (m.matches())
- this.colParsers.add(new FlowDepthColumnParser(this.importPath, this.rootRelativePath, this.river, this.tkhGroup, i, m.group(4).trim()));
+ this.colParsers.add(new FlowDepthColumnParser(this.importPath, this.rootRelativePath, this.river, this.flowdepthGroup, i, m.group(4).trim()));
else
- logWarning("No title found in column " + i + ", skipped");
+ logLineWarning("Invalid title/unit in column %d (%s)", i + 1, this.columnTitles.get(i));
}
return true;
}
+ /**
+ * Checks the existence of the active series in the database
+ */
+ @Override
+ protected boolean checkSeriesExistsAlready() {
+ if (!checkRiverExists())
+ return false;
+ final Session session = ImporterSession.getInstance().getDatabaseSession();
+ final List<FlowDepth> rows = this.flowdepthGroup.querySeriesItem(session, this.river.getPeer());
+ return !rows.isEmpty();
+ }
+
+
@Override
public void store() {
if (this.headerParsingState != ParsingState.STOP) {
- this.tkhGroup.getPeer(this.river.getPeer());
+ this.flowdepthGroup.getPeer(this.river.getPeer());
for (final FlowDepthColumnParser colParser : this.colParsers)
colParser.store();
}
else
- logWarning("Severe parsing errors, not storing series '" + this.tkhGroup.getFilename() + "'");
+ logWarning("Severe parsing errors, not storing series '%s'", this.flowdepthGroup.getFilename());
}
@Override
More information about the Dive4Elements-commits
mailing list