[PATCH 06 of 12] Importer (s/u-info) extensions: uniform formatting of from-to series names,
Wald Commits
scm-commit at wald.intevation.org
Mon Mar 23 16:38:43 CET 2020
# HG changeset patch
# User mschaefer
# Date 1584972995 -3600
# Mon Mar 23 15:16:35 2020 +0100
# Node ID 1f57381b3bb579913a3742216899abe83fc888c7
# Parent 8a2a777a8372c57343c84427381de4b4c51788dd
Importer (s/u-info) extensions: uniform formatting of from-to series names,
warning instead of cancelling in case of missing tkh column values,
detecting, logging and skipping columns with wrong unit
diff -r 8a2a777a8372 -r 1f57381b3bb5 backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhColumnSeriesImport.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhColumnSeriesImport.java Mon Mar 23 15:10:09 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhColumnSeriesImport.java Mon Mar 23 15:16:35 2020 +0100
@@ -55,7 +55,11 @@
public TkhColumnSeriesImport(final String filename, final TkhSeriesImport parent, final String colName, final File relativeFilePath) {
super(filename);
this.parent = parent;
- this.colName = colName;
+ final String[] items = (colName == null) ? new String[] {} : colName.split("\\-");
+ if (items.length == 2)
+ this.colName = items[0].trim() + " - " + items[1].trim();
+ else
+ this.colName = colName;
this.relativeFilePath = relativeFilePath;
}
@@ -91,14 +95,22 @@
return log;
}
- @Override
- public List<TkhColumn> querySeriesItem(final Session session, final River river) {
+ private List<TkhColumn> querySeriesItem(final Session session, final River river) {
final Query query = session.createQuery("FROM TkhColumn WHERE tkh=:parent AND lower(name)=:colname");
query.setParameter("parent", this.parent.getPeer(river));
query.setParameter("colname", this.colName.toLowerCase());
return query.list();
}
+ @Override
+ public List<TkhColumn> querySeriesItem(final Session session, final River river, final boolean doQueryParent) {
+ final Query query = session.createQuery("FROM TkhColumn c INNER JOIN c.tkh s"
+ + " WHERE s.river=:river AND lower(s.filename)=:filename AND lower(c.name)=:colname");
+ query.setParameter("river", river);
+ query.setParameter("filename", this.filename.toLowerCase());
+ query.setParameter("colname", this.colName.toLowerCase());
+ return query.list();
+ }
@Override
public TkhColumn createSeriesItem(final River river) {
diff -r 8a2a777a8372 -r 1f57381b3bb5 backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhKmLineImport.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhKmLineImport.java Mon Mar 23 15:10:09 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhKmLineImport.java Mon Mar 23 15:16:35 2020 +0100
@@ -15,7 +15,7 @@
import org.dive4elements.river.importer.common.AbstractKmLineImport;
import org.dive4elements.river.model.sinfo.TkhColumn;
import org.dive4elements.river.model.sinfo.TkhValue;
-import org.hibernate.SQLQuery;
+import org.hibernate.Query;
import org.hibernate.Session;
/**
@@ -46,15 +46,10 @@
@Override
protected TkhValue queryValueItem(final Session session, final TkhColumn parent) {
- /*
- * final Query query = session.createQuery("FROM TkhValue WHERE (TkhColumn=:parent)"
- * + " AND (station BETWEEN (:station-0.0001) AND (:station+0.0001))");
- * query.setParameter("parent", parent);
- * query.setParameter("station", this.station);
- */
- final SQLQuery query = session.createSQLQuery("SELECT * FROM tkh_values WHERE (tkh_column_id=:parent)"
+ final Query query = session.createQuery("FROM TkhValue WHERE (tkhColumn=:parent)"
+ " AND (station BETWEEN (:station-0.0001) AND (:station+0.0001))");
- query.setParameter("parent", parent.getId()).setParameter("station", this.station);
+ query.setParameter("parent", parent);
+ query.setParameter("station", this.station);
final List rows = query.list();
if (!rows.isEmpty())
return (TkhValue) rows.get(0);
diff -r 8a2a777a8372 -r 1f57381b3bb5 backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhSeriesImport.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhSeriesImport.java Mon Mar 23 15:10:09 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/importitem/TkhSeriesImport.java Mon Mar 23 15:16:35 2020 +0100
@@ -144,7 +144,7 @@
return this.peer;
}
- private List<Tkh> querySeriesItem(final Session session, final River river) {
+ public List<Tkh> querySeriesItem(final Session session, final River river) {
final Query query = session.createQuery("FROM Tkh WHERE river=:river AND lower(filename)=:filename");
query.setParameter("river", river);
query.setParameter("filename", this.filename.toLowerCase());
diff -r 8a2a777a8372 -r 1f57381b3bb5 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhColumnParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhColumnParser.java Mon Mar 23 15:10:09 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhColumnParser.java Mon Mar 23 15:16:35 2020 +0100
@@ -61,7 +61,7 @@
@Override
protected void logStartInfo() {
- getLog().info(String.format("Start parsing column %d '%s':;'%s'", this.colIndex, this.colName, this.rootRelativePath));
+ getLog().info(String.format("Start parsing column %d '%s':;'%s'", this.colIndex + 1, this.colName, this.rootRelativePath));
}
@Override
@@ -77,8 +77,13 @@
@Override
protected TkhKmLineImport createKmLineImport(final Double km, final String[] values) {
+ final Number value = parseDoubleCheckNull(values, this.colIndex);
+ if ((value == null) || Double.isNaN(value.doubleValue())) {
+ logLineWarning("Column %d: " + INVALID_VALUE_ERROR_FORMAT, this.colIndex + 1, "TKH");
+ return null;
+ }
// convert from cm to m
- final double tkheight = parseDoubleWithNull(values[this.colIndex]).doubleValue() / 100.0;
- return new TkhKmLineImport(km, tkheight);
+ final double tkh = value.doubleValue() / 100.0;
+ return new TkhKmLineImport(km, tkh);
}
}
diff -r 8a2a777a8372 -r 1f57381b3bb5 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhParser.java Mon Mar 23 15:10:09 2020 +0100
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhParser.java Mon Mar 23 15:16:35 2020 +0100
@@ -12,7 +12,6 @@
import java.io.File;
import java.io.FileInputStream;
-import java.io.IOException;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.text.DateFormat;
@@ -26,6 +25,7 @@
import org.apache.log4j.Logger;
import org.dive4elements.river.importer.Config;
import org.dive4elements.river.importer.ImportRiver;
+import org.dive4elements.river.importer.ImporterSession;
import org.dive4elements.river.importer.common.AbstractParser;
import org.dive4elements.river.importer.common.ParsingState;
import org.dive4elements.river.importer.sinfo.importitem.TkhColumnSeriesImport;
@@ -34,6 +34,7 @@
import org.dive4elements.river.model.sinfo.Tkh;
import org.dive4elements.river.model.sinfo.TkhColumn;
import org.dive4elements.river.model.sinfo.TkhValue;
+import org.hibernate.Session;
/**
* Reads and parses the header of a TKH file and handles the parse and store of the columns
@@ -131,7 +132,7 @@
}
@Override
- public void parse() throws IOException {
+ public void parse() throws Exception {
getLog().info("Start parsing:;'" + this.rootRelativePath + "'");
// this.seriesHeader = createSeriesImport(this.importPath.getName().replaceAll("\\.csv", ""));
this.metaPatternsMatched.clear();
@@ -143,7 +144,7 @@
this.in = new LineNumberReader(new InputStreamReader(new FileInputStream(this.importPath), ENCODING));
}
catch (final Exception e) {
- logError("Could not open (" + e.getMessage() + ")");
+ logError("Could not open (%s)", e.getMessage());
this.headerParsingState = ParsingState.STOP;
}
this.currentLine = null;
@@ -155,6 +156,8 @@
if (this.currentLine.isEmpty())
continue;
handleMetaLine();
+ if (this.headerParsingState == ParsingState.DONE)
+ checkMetaData();
}
}
finally {
@@ -194,12 +197,12 @@
day = dateFormat.parse(m.group(1));
}
catch (final Exception e) {
- logError("Invalid date in line " + this.in.getLineNumber());
+ logLineWarning("Invalid date");
}
this.metaPatternsMatched.add(META_DATUM);
this.tkhGroup.setDay(day);
if (day == null)
- logWarning("No date specified");
+ logLineWarning("No date specified");
return true;
}
return false;
@@ -241,11 +244,24 @@
if (m.matches())
this.colParsers.add(new TkhColumnParser(this.importPath, this.rootRelativePath, this.river, this.tkhGroup, i, m.group(7).trim()));
else
- logWarning("No title found in column " + i + ", skipped");
+ logLineWarning("Invalid title in column %d (%s)", i + 1, this.columnTitles.get(i));
}
return true;
}
+ /**
+ * Checks the existence of the active series in the database
+ */
+ @Override
+ protected boolean checkSeriesExistsAlready() {
+ if (!checkRiverExists())
+ return false;
+ final Session session = ImporterSession.getInstance().getDatabaseSession();
+ final List<Tkh> rows = this.tkhGroup.querySeriesItem(session, this.river.getPeer());
+ return !rows.isEmpty();
+ }
+
+
@Override
public void store() {
if (this.headerParsingState != ParsingState.STOP) {
@@ -254,7 +270,7 @@
colParser.store();
}
else
- logWarning("Severe parsing errors, not storing series '" + this.tkhGroup.getFilename() + "'");
+ logWarning("Severe parsing errors, not storing series '%s'", this.tkhGroup.getFilename());
}
@Override
More information about the Dive4Elements-commits
mailing list