[PATCH] SCHEMA CHANGE and Importer (only longitudinal section data so far): only grain fractions given now in schema are valid. We are handling sediment loads, not yields
Wald Commits
scm-commit at wald.intevation.org
Mon Jul 14 15:36:51 CEST 2014
# HG changeset patch
# User "Tom Gottfried <tom at intevation.de>"
# Date 1405345004 -7200
# Node ID fd3a24336e6a9f454e2bf21b78decf00119b70cc
# Parent 6954ac9b7591ab2c58b899838c3a73cce6ea6112
SCHEMA CHANGE and Importer (only longitudinal section data so far): only grain fractions given now in schema are valid. We are handling sediment loads, not yields.
diff -r 6954ac9b7591 -r fd3a24336e6a backend/contrib/import_river.sh
--- a/backend/contrib/import_river.sh Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/contrib/import_river.sh Mon Jul 14 15:36:44 2014 +0200
@@ -235,7 +235,7 @@
-Dflys.backend.importer.skip.waterlevels=true \
-Dflys.backend.importer.skip.sq.relation=true \
-Dflys.backend.importer.skip.sediment.density=true \
- -Dflys.backend.importer.skip.sediment.yield=true \
+ -Dflys.backend.importer.skip.sediment.load.ls=true \
-Dflys.backend.importer.skip.morphological.width=true \
-Dflys.backend.importer.skip.flow.velocity=true \
-Dflys.backend.importer.skip.bed.height.single=true \
@@ -282,7 +282,7 @@
-Dflys.backend.importer.skip.waterlevels=false \
-Dflys.backend.importer.skip.sq.relation=false \
-Dflys.backend.importer.skip.sediment.density=false \
- -Dflys.backend.importer.skip.sediment.yield=false \
+ -Dflys.backend.importer.skip.sediment.load.ls=false \
-Dflys.backend.importer.skip.morphological.width=false \
-Dflys.backend.importer.skip.flow.velocity=false \
-Dflys.backend.importer.skip.bed.height.single=false \
@@ -329,7 +329,7 @@
-Dflys.backend.importer.skip.waterlevels=true \
-Dflys.backend.importer.skip.sq.relation=true \
-Dflys.backend.importer.skip.sediment.density=true \
- -Dflys.backend.importer.skip.sediment.yield=true \
+ -Dflys.backend.importer.skip.sediment.load.ls=true \
-Dflys.backend.importer.skip.morphological.width=true \
-Dflys.backend.importer.skip.flow.velocity=true \
-Dflys.backend.importer.skip.bed.height.single=true \
diff -r 6954ac9b7591 -r fd3a24336e6a backend/contrib/run_hydr_morph.sh
--- a/backend/contrib/run_hydr_morph.sh Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/contrib/run_hydr_morph.sh Mon Jul 14 15:36:44 2014 +0200
@@ -47,7 +47,7 @@
IMPORTER_SKIP_MORPHOLOGICAL_WIDTH=false
IMPORTER_SKIP_POROSITY=false
IMPORTER_SKIP_SEDIMENT_DENSITY=false
-IMPORTER_SKIP_SEDIMENT_YIELD=false
+IMPORTER_SKIP_SEDIMENT_LOAD_LS=false
IMPORTER_SKIP_SQ_RELATION=false
IMPORTER_SKIP_WATERLEVELS=false
IMPORTER_SKIP_WATERLEVEL_DIFFERENCES=false
@@ -95,7 +95,7 @@
-Dflys.backend.importer.skip.official.lines=$IMPORTER_SKIP_OFFICIAL_LINES \
-Dflys.backend.importer.skip.prfs=$IMPORTER_SKIP_PRFS \
-Dflys.backend.importer.skip.sediment.density=$IMPORTER_SKIP_SEDIMENT_DENSITY \
- -Dflys.backend.importer.skip.sediment.yield=$IMPORTER_SKIP_SEDIMENT_YIELD \
+ -Dflys.backend.importer.skip.sediment.load.ls=$IMPORTER_SKIP_SEDIMENT_LOAD_LS \
-Dflys.backend.importer.skip.sq.relation=$IMPORTER_SKIP_SQ_RELATION \
-Dflys.backend.importer.skip.w80s=$IMPORTER_SKIP_W80S \
-Dflys.backend.importer.skip.w80.csvs=$IMPORTER_SKIP_W80_CSVS \
diff -r 6954ac9b7591 -r fd3a24336e6a backend/doc/documentation/de/importer-hydr-morph.tex
--- a/backend/doc/documentation/de/importer-hydr-morph.tex Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/doc/documentation/de/importer-hydr-morph.tex Mon Jul 14 15:36:44 2014 +0200
@@ -211,13 +211,18 @@
\textit{Morphologie/Geschwindigkeit\_Schubspannung/Modellrechnungen} und\\
\textit{Morphologie/Geschwindigkeit\_Schubspannung/v-Messungen} geladen.
-\subsubsection{Sedimentfracht}
-Der Import der Sedimentfracht kann mit
-\textbf{-Dflys.backend.importer.skip.sediment.yield=true}
-unterdrückt werden. Es werden die CSV-Dateien aus dem Verzeichnis
-\textit{Morphologie/Fracht} geladen. Dabei werden die Dateien aus dem
-Unterverzeichnissen \textit{Einzeljahre} und \textit{Epochen} entsprechend als
-\textit{Einzeljahre} und \textit{Epochen} gespeichert.
+\subsubsection{Sedimentfracht (Längsschnitt-Daten)}
+Mit
+\textbf{-Dflys.backend.importer.skip.sediment.load.ls=true}
+kann der Import der Längsschnitt-Daten (auf freier Strecke)
+der Sedimentfracht unterdrückt werden.
+Es werden die CSV-Dateien aus dem Verzeichnis
+\textit{Morphologie/Fracht/Laengsschnitte} geladen.
+Dabei werden die Dateien aus dem
+Unterverzeichnissen \textit{Einzeljahre}, \textit{Epochen}
+und \textit{amtliche Epochen} entsprechend als
+\textit{Einzeljahre}, \textit{Epochen} und
+\textit{amtliche Epochen} gespeichert.
\subsubsection{Wasserspiegellagen für M-INFO}
Mit \textbf{-Dflys.backend.importer.skip.waterlevels=true}
diff -r 6954ac9b7591 -r fd3a24336e6a backend/doc/schema/oracle-minfo.sql
--- a/backend/doc/schema/oracle-minfo.sql Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/doc/schema/oracle-minfo.sql Mon Jul 14 15:36:44 2014 +0200
@@ -243,8 +243,21 @@
name VARCHAR(64) NOT NULL,
lower NUMBER(38,3),
upper NUMBER(38,3),
- PRIMARY KEY (id)
+ PRIMARY KEY (id),
+ UNIQUE(name, lower, upper)
);
+-- single fractions
+INSERT INTO grain_fraction VALUES (1, 'coarse', 16, 200);
+INSERT INTO grain_fraction VALUES (2, 'fine_middle', 2, 16);
+INSERT INTO grain_fraction VALUES (3, 'sand', 0.063, 2);
+INSERT INTO grain_fraction VALUES (4, 'susp_sand', 0.063, 2);
+INSERT INTO grain_fraction VALUES (5, 'susp_sand_bed', 0.063, 2);
+INSERT INTO grain_fraction VALUES (6, 'suspended_sediment', 0, 0.063);
+-- aggregations of fractions
+INSERT INTO grain_fraction VALUES (7, 'total', 0, 200);
+INSERT INTO grain_fraction VALUES (8, 'bed_load', 0.063, 200);
+INSERT INTO grain_fraction VALUES (9, 'suspended_load', 0, 2);
+
--lookup table for sediment yield kinds
CREATE TABLE sediment_yield_kinds (
diff -r 6954ac9b7591 -r fd3a24336e6a backend/doc/schema/postgresql-minfo.sql
--- a/backend/doc/schema/postgresql-minfo.sql Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/doc/schema/postgresql-minfo.sql Mon Jul 14 15:36:44 2014 +0200
@@ -243,8 +243,20 @@
name VARCHAR(64) NOT NULL,
lower NUMERIC,
upper NUMERIC,
- PRIMARY KEY (id)
+ PRIMARY KEY (id),
+ UNIQUE(name, lower, upper)
);
+-- single fractions
+INSERT INTO grain_fraction VALUES (1, 'coarse', 16, 200);
+INSERT INTO grain_fraction VALUES (2, 'fine_middle', 2, 16);
+INSERT INTO grain_fraction VALUES (3, 'sand', 0.063, 2);
+INSERT INTO grain_fraction VALUES (4, 'susp_sand', 0.063, 2);
+INSERT INTO grain_fraction VALUES (5, 'susp_sand_bed', 0.063, 2);
+INSERT INTO grain_fraction VALUES (6, 'suspended_sediment', 0, 0.063);
+-- aggregations of fractions
+INSERT INTO grain_fraction VALUES (7, 'total', 0, 200);
+INSERT INTO grain_fraction VALUES (8, 'bed_load', 0.063, 200);
+INSERT INTO grain_fraction VALUES (9, 'suspended_load', 0, 2);
--lookup table for sediment yield kinds
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/importer/Config.java
--- a/backend/src/main/java/org/dive4elements/river/importer/Config.java Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/Config.java Mon Jul 14 15:36:44 2014 +0200
@@ -85,8 +85,8 @@
public static final String SKIP_FLOW_VELOCITY =
"flys.backend.importer.skip.flow.velocity";
- public static final String SKIP_SEDIMENT_YIELD =
- "flys.backend.importer.skip.sediment.yield";
+ public static final String SKIP_SEDIMENT_LOAD_LS =
+ "flys.backend.importer.skip.sediment.load.ls";
public static final String SKIP_WATERLEVELS =
"flys.backend.importer.skip.waterlevels";
@@ -227,8 +227,8 @@
return getFlag(SKIP_FLOW_VELOCITY);
}
- public boolean skipSedimentYield() {
- return getFlag(SKIP_SEDIMENT_YIELD);
+ public boolean skipSedimentLoadLS() {
+ return getFlag(SKIP_SEDIMENT_LOAD_LS);
}
public boolean skipWaterlevels() {
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/importer/ImportGrainFraction.java
--- a/backend/src/main/java/org/dive4elements/river/importer/ImportGrainFraction.java Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/ImportGrainFraction.java Mon Jul 14 15:36:44 2014 +0200
@@ -13,24 +13,10 @@
public class ImportGrainFraction {
- private String name;
+ private GrainFraction peer;
- private Double lower;
- private Double upper;
-
- public ImportGrainFraction(String name) {
- this.name = name;
- }
-
-
- public ImportGrainFraction(
- String name,
- Double lower,
- Double upper
- ) {
- this.name = name;
- this.lower = lower;
- this.upper = upper;
+ public ImportGrainFraction(GrainFraction gf) {
+ this.peer = gf;
}
@@ -40,7 +26,7 @@
public GrainFraction getPeer() {
- return ImporterSession.getInstance().getGrainFraction(name);
+ return peer;
}
}
// vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/importer/ImportRiver.java
--- a/backend/src/main/java/org/dive4elements/river/importer/ImportRiver.java Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/ImportRiver.java Mon Jul 14 15:36:44 2014 +0200
@@ -104,13 +104,15 @@
public static final String FLOW_VELOCITY_MEASUREMENTS = "v-Messungen";
- public static final String SEDIMENT_YIELD_DIR = "Fracht";
+ public static final String SEDIMENT_LOAD_DIR = "Fracht";
- public static final String SEDIMENT_YIELD_SINGLE_DIR = "Einzeljahre";
+ public static final String SEDIMENT_LOAD_LS_DIR = "Laengsschnitte";
- public static final String SEDIMENT_YIELD_EPOCH_DIR = "Epochen";
+ public static final String SEDIMENT_LOAD_SINGLE_DIR = "Einzeljahre";
- public static final String SEDIMENT_YIELD_OFF_EPOCH_DIR = "amtliche Epochen";
+ public static final String SEDIMENT_LOAD_EPOCH_DIR = "Epochen";
+
+ public static final String SEDIMENT_LOAD_OFF_EPOCH_DIR = "amtliche Epochen";
public static final String MINFO_FIXATIONS_DIR = "Fixierungsanalyse";
@@ -172,7 +174,7 @@
protected List<ImportFlowVelocityMeasurement> flowVelocityMeasurements;
- protected List<ImportSedimentLoadLS> sedimentYields;
+ protected List<ImportSedimentLoadLS> sedimentLoadLSs;
protected List<ImportMeasurementStation> measurementStations;
@@ -288,7 +290,7 @@
morphologicalWidths = new ArrayList<ImportMorphWidth>();
flowVelocityModels = new ArrayList<ImportFlowVelocityModel>();
flowVelocityMeasurements = new ArrayList<ImportFlowVelocityMeasurement>();
- sedimentYields = new ArrayList<ImportSedimentLoadLS>();
+ sedimentLoadLSs = new ArrayList<ImportSedimentLoadLS>();
measurementStations = new ArrayList<ImportMeasurementStation>();
sqRelations = new ArrayList<ImportSQRelation>();
}
@@ -381,7 +383,7 @@
parsePorosity();
parseMorphologicalWidth();
parseFlowVelocity();
- parseSedimentYield();
+ parseSedimentLoadLS();
parseWaterlevels();
parseWaterlevelDifferences();
parseMeasurementStations();
@@ -593,7 +595,7 @@
}
- private void parseSedimentYieldDir(
+ private void parseSedimentLoadLSDir(
File[] files,
SedimentLoadLSParser parser
) throws IOException {
@@ -609,20 +611,25 @@
}
}
- protected void parseSedimentYield() throws IOException {
- if (Config.INSTANCE.skipSedimentYield()) {
- log.info("skip parsing sediment yield data");
+ protected void parseSedimentLoadLS() throws IOException {
+ if (Config.INSTANCE.skipSedimentLoadLS()) {
+ log.info("skip parsing sediment load longitudinal section data");
return;
}
- log.debug("Parse sediment yield data");
+ log.debug("Parse sediment load longitudinal section data");
- File minfoDir = getMinfoDir();
- File sedimentYieldDir = new File(minfoDir, SEDIMENT_YIELD_DIR);
+ File minfoDir = getMinfoDir();
+ File sedimentLoadDir = new File(minfoDir, SEDIMENT_LOAD_DIR);
+ File sedimentLoadLSDir = new File(sedimentLoadDir,
+ SEDIMENT_LOAD_LS_DIR);
- File singleDir = new File(sedimentYieldDir, SEDIMENT_YIELD_SINGLE_DIR);
- File epochDir = new File(sedimentYieldDir, SEDIMENT_YIELD_EPOCH_DIR);
- File offEpochDir = new File(sedimentYieldDir, SEDIMENT_YIELD_OFF_EPOCH_DIR);
+ File singleDir = new File(sedimentLoadLSDir,
+ SEDIMENT_LOAD_SINGLE_DIR);
+ File epochDir = new File(sedimentLoadLSDir,
+ SEDIMENT_LOAD_EPOCH_DIR);
+ File offEpochDir = new File(sedimentLoadLSDir,
+ SEDIMENT_LOAD_OFF_EPOCH_DIR);
File[] singles = singleDir.listFiles();
File[] epochs = epochDir.listFiles();
@@ -634,24 +641,24 @@
log.warn("Cannot read directory '" + singleDir + "'");
}
else {
- parseSedimentYieldDir(singles, parser);
+ parseSedimentLoadLSDir(singles, parser);
}
if (epochs == null || epochs.length == 0) {
log.warn("Cannot read directory '" + epochDir + "'");
}
else {
- parseSedimentYieldDir(epochs, parser);
+ parseSedimentLoadLSDir(epochs, parser);
}
if (offEpochs == null || offEpochs.length == 0) {
log.warn("Cannot read directory '" + offEpochDir + "'");
}
else {
- parseSedimentYieldDir(offEpochs, parser);
+ parseSedimentLoadLSDir(offEpochs, parser);
}
- sedimentYields = parser.getSedimentYields();
+ sedimentLoadLSs = parser.getSedimentLoadLSs();
}
@@ -1255,7 +1262,7 @@
storePorosity();
storeMorphologicalWidth();
storeFlowVelocity();
- storeSedimentYield();
+ storeSedimentLoadLS();
storeWaterlevels();
storeWaterlevelDifferences();
storeMeasurementStations();
@@ -1495,14 +1502,14 @@
}
- public void storeSedimentYield() {
- if (!Config.INSTANCE.skipSedimentYield()) {
- log.info("store sediment yield data");
+ public void storeSedimentLoadLS() {
+ if (!Config.INSTANCE.skipSedimentLoadLS()) {
+ log.info("store sediment load longitudinal section data");
River river = getPeer();
- for (ImportSedimentLoadLS sedimentYield: sedimentYields) {
- sedimentYield.storeDependencies(river);
+ for (ImportSedimentLoadLS sedimentLoadLS: sedimentLoadLSs) {
+ sedimentLoadLS.storeDependencies(river);
}
}
}
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/importer/ImportSedimentLoadLS.java
--- a/backend/src/main/java/org/dive4elements/river/importer/ImportSedimentLoadLS.java Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/ImportSedimentLoadLS.java Mon Jul 14 15:36:44 2014 +0200
@@ -83,7 +83,7 @@
i++;
}
- log.info("stored " + i + " sediment yield values.");
+ log.info("stored " + i + " sediment load values.");
}
}
@@ -98,14 +98,14 @@
TimeInterval ti = timeInterval != null ? timeInterval.getPeer() : null;
if (ti == null || u == null) {
- log.warn("Skip invalid SedimentYield: time interval or unit null!");
+ log.warn("Skip invalid SedimentLoadLS: time interval or unit null!");
return null;
}
if (peer == null) {
Session session = ImporterSession.getInstance()
.getDatabaseSession();
- Query query = session.createQuery("from SedimentYield where "
+ Query query = session.createQuery("from SedimentLoadLS where "
+ " river=:river and "
+ " grainFraction=:grainFraction and " + " unit=:unit and "
+ " timeInterval=:timeInterval and "
@@ -117,16 +117,16 @@
query.setParameter("timeInterval", ti);
query.setParameter("description", description);
- List<SedimentLoadLS> yields = query.list();
- if (yields.isEmpty()) {
- log.debug("create new SedimentYield");
+ List<SedimentLoadLS> loads = query.list();
+ if (loads.isEmpty()) {
+ log.debug("create new SedimentLoadLS");
peer = new SedimentLoadLS(river, u, ti, gf, description);
peer.setKind(this.kind);
session.save(peer);
}
else {
- peer = yields.get(0);
+ peer = loads.get(0);
}
}
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/importer/ImportSedimentLoadLSValue.java
--- a/backend/src/main/java/org/dive4elements/river/importer/ImportSedimentLoadLSValue.java Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/ImportSedimentLoadLSValue.java Mon Jul 14 15:36:44 2014 +0200
@@ -31,28 +31,28 @@
}
- public void storeDependencies(SedimentLoadLS sedimentYield) {
- getPeer(sedimentYield);
+ public void storeDependencies(SedimentLoadLS sedimentLoadLS) {
+ getPeer(sedimentLoadLS);
}
- public SedimentLoadLSValue getPeer(SedimentLoadLS sedimentYield) {
+ public SedimentLoadLSValue getPeer(SedimentLoadLS sedimentLoadLS) {
if (peer == null) {
Session session = ImporterSession.getInstance().getDatabaseSession();
Query query = session.createQuery(
- "from SedimentYieldValue where " +
- " sedimentYield=:sedimentYield and " +
+ "from SedimentLoadLSValue where " +
+ " sedimentLoadLS=:sedimentLoadLS and " +
" station=:station and " +
" value=:value"
);
- query.setParameter("sedimentYield", sedimentYield);
+ query.setParameter("sedimentLoadLS", sedimentLoadLS);
query.setParameter("station", station);
query.setParameter("value", value);
List<SedimentLoadLSValue> values = query.list();
if (values.isEmpty()) {
- peer = new SedimentLoadLSValue(sedimentYield, station, value);
+ peer = new SedimentLoadLSValue(sedimentLoadLS, station, value);
session.save(peer);
}
else {
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/importer/parsers/LineParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/parsers/LineParser.java Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/parsers/LineParser.java Mon Jul 14 15:36:44 2014 +0200
@@ -113,12 +113,8 @@
protected static String stripMetaLine(String line) {
String tmp = line.substring(1, line.length());
- if (tmp.startsWith(" ")) {
- return tmp.substring(1, tmp.length());
- }
- else {
- return tmp;
- }
+ // meta-lines often have trailing semicolons in real data
+ return tmp.replaceAll(SEPERATOR_CHAR + "*$", "").trim();
}
public static double getDouble(String doubleString) throws ParseException {
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/importer/parsers/SedimentLoadLSParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/parsers/SedimentLoadLSParser.java Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/parsers/SedimentLoadLSParser.java Mon Jul 14 15:36:44 2014 +0200
@@ -21,16 +21,19 @@
import org.apache.log4j.Logger;
+import org.dive4elements.river.importer.ImporterSession;
import org.dive4elements.river.importer.ImportGrainFraction;
import org.dive4elements.river.importer.ImportSedimentLoadLS;
import org.dive4elements.river.importer.ImportSedimentLoadLSValue;
import org.dive4elements.river.importer.ImportTimeInterval;
import org.dive4elements.river.importer.ImportUnit;
+
import org.dive4elements.river.model.GrainFraction;
+
import org.dive4elements.river.utils.DateUtil;
+import org.dive4elements.river.utils.EpsilonComparator;
-
-/** Parses Sediment Yield files. */
+/** Parses sediment load longitudinal section files. */
public class SedimentLoadLSParser extends LineParser {
private static final Logger log =
@@ -40,33 +43,6 @@
public static final NumberFormat nf = NumberFormat.getInstance(DEFAULT_LOCALE);
- public static final String FRAKTION_START = "Fraktion:";
-
- public static final String FRACTION_COARSE_STR =
- ".*Grobkorn.*";
-
- public static final String FRACTION_FINE_MIDDLE_STR =
- ".*Fein.Mittel.Kies.*";
-
- public static final String FRACTION_SAND =
- ".*Sand.*";
-
- public static final String FRACTION_SUSP_SAND =
- ".*susp.Sand.*";
-
- public static final String FRACTION_SUSP_SAND_BED =
- ".*bettbild.Anteil.susp.Sand.*";
-
- public static final String FRACTION_SUSP_SAND_BED_EPOCH =
- ".*susp.Sand.bettbildAnteil.*";
-
- public static final String FRACTION_SUSPENDED_SEDIMENT =
- ".*Schwebstoff.*";
-
- public static final String FRACTION_TOTAL =
- ".*gesamt.*";
-
-
public static final Pattern TIMEINTERVAL_SINGLE =
Pattern.compile("\\D*([0-9]+?)\\D*");
@@ -76,23 +52,20 @@
public static final Pattern META_FRACTION =
Pattern.compile("^Fraktion: (.*)");
+ public static final Pattern META_FRACTION_NAME =
+ Pattern.compile("^Fraktionsname: (.*)");
+
public static final Pattern META_UNIT =
Pattern.compile("^Einheit: \\[(.*)\\].*");
public static final Pattern META_COLUMN_NAMES =
Pattern.compile("^Fluss-km.*");
- public static final Pattern META_GRAIN_FRACTION_A =
- Pattern.compile("\\D*(([0-9]+?,[0-9]+?)\\s*-|([0-9]++)\\s*-)(([0-9]+?,[0-9]+?)|([0-9]++))\\s*([a-zA-Z]+?)\\W*\\D*");
+ public static final Pattern META_GRAIN_SIZE =
+ Pattern.compile("([0-9]*,*[0-9]+)-([0-9]*,*[0-9]+) *mm");
- public static final Pattern META_GRAIN_FRACTION_B =
- Pattern.compile("(<|>){1}\\s*(\\w++)\\s*(([0-9]+?,[0-9]+?)\\s*-|([0-9]++)\\s*-)(([0-9]+?,[0-9]+?)|([0-9]++))\\s*([a-zA-Z]+?)");
- public static final Pattern META_GRAIN_FRACTION_C =
- Pattern.compile("(<|>){1}\\s*((([0-9]+?,[0-9]+?)|([0-9]++))\\s*(\\w+))");
-
-
- protected List<ImportSedimentLoadLS> sedimentYields;
+ protected List<ImportSedimentLoadLS> sedimentLoadLSs;
protected ImportSedimentLoadLS[] current;
@@ -104,9 +77,13 @@
protected String[] columnNames;
+ private String upper;
+
+ private String lower;
+
public SedimentLoadLSParser() {
- sedimentYields = new ArrayList<ImportSedimentLoadLS>();
+ sedimentLoadLSs = new ArrayList<ImportSedimentLoadLS>();
}
@@ -130,7 +107,7 @@
protected void finish() {
if (current != null) {
for (ImportSedimentLoadLS isy: current) {
- sedimentYields.add(isy);
+ sedimentLoadLSs.add(isy);
}
}
@@ -156,11 +133,14 @@
else if (handleMetaFraction(line)) {
return;
}
+ else if (handleMetaFractionName(line)) {
+ return;
+ }
else if (handleColumnNames(line)) {
return;
}
else {
- log.warn("SYP: Unknown meta line: '" + line + "'");
+ log.warn("SLLSP: Unknown meta line: '" + line + "'");
}
}
@@ -181,20 +161,60 @@
Matcher m = META_FRACTION.matcher(line);
if (m.matches()) {
- String tmp = m.group(1);
+ String interval = m.group(1);
- this.grainFraction = buildGrainFraction(tmp);
-
- return true;
- }
- else if (line.startsWith(FRAKTION_START)) {
- String newLine = line.replace(FRAKTION_START, "").trim();
- if (newLine.length() == 0) {
- log.debug("Found total grain fraction.");
- this.grainFraction = new ImportGrainFraction(GrainFraction.UNKNOWN);
+ Matcher sizes = META_GRAIN_SIZE.matcher(interval);
+ if (sizes.matches()) {
+ lower = sizes.group(1);
+ upper = sizes.group(2);
return true;
}
+
+ log.warn("SLLSP: Unrecognized grain-size interval. Ignored.");
+ return true;
+
+ }
+
+ return false;
+ }
+
+
+ public boolean handleMetaFractionName(String line) {
+ Matcher m = META_FRACTION_NAME.matcher(line);
+
+ if (m.matches()) {
+ String name = m.group(1);
+
+
+ GrainFraction gf = ImporterSession.getInstance().getGrainFraction(name);
+
+ if (gf != null) {
+
+ if (lower != null && upper != null) {
+ // Validate grain size interval
+ try {
+ Double lowval = nf.parse(lower).doubleValue();
+ Double upval = nf.parse(upper).doubleValue();
+
+ if (EpsilonComparator.CMP.compare(lowval,
+ gf.getLower()) != 0 ||
+ EpsilonComparator.CMP.compare(upval,
+ gf.getUpper()) != 0) {
+ log.warn("SLLSP: Invalid grain size for grain fraction '" +
+ name + "'. Ignored.");
+ }
+ }
+ catch (ParseException pe) {
+ log.warn("SLLSP: Could not parse grain-size interval. Ignored.");
+ }
+ }
+
+ grainFraction = new ImportGrainFraction(gf);
+ return true;
+ }
+
+ log.error("SLLSP: Unknown grain fraction: '" + name + "'");
}
return false;
@@ -207,7 +227,13 @@
if (m.matches()) {
columnNames = line.split(SEPERATOR_CHAR);
- initializeSedimentYields();
+ // 'Fluss-km', 'Hinweise' and at least one data column required
+ if (columnNames.length < 3) {
+ log.error("SLLSP: missing columns.");
+ return true;
+ }
+
+ initializeSedimentLoadLSs();
return true;
}
@@ -220,7 +246,7 @@
String[] vals = line.split(SEPERATOR_CHAR);
if (vals == null || vals.length < columnNames.length-1) {
- log.warn("SYP: skip invalid data line: '" + line + "'");
+ log.warn("SLLSP: skip invalid data line: '" + line + "'");
return;
}
@@ -238,14 +264,14 @@
}
}
catch (ParseException pe) {
- log.warn("SYP: unparseable number in data row '" + line + "':", pe);
+ log.warn("SLLSP: unparseable number in data row '" + line + "':", pe);
}
}
- /** Initialize SedimentYields from columns, set the kind
+ /** Initialize SedimentLoadLSs from columns, set the kind
* with respect to file location (offical epoch or not?) */
- private void initializeSedimentYields() {
+ private void initializeSedimentLoadLSs() {
// skip first column (Fluss-km) and last column (Hinweise)
current = new ImportSedimentLoadLS[columnNames.length-2];
@@ -288,127 +314,18 @@
return new ImportTimeInterval(DateUtil.getStartDateFromYear(year));
}
- log.warn("SYP: Unknown time interval string: '" + column + "'");
+ log.warn("SLLSP: Unknown time interval string: '" + column + "'");
}
catch (ParseException pe) {
- log.warn("SYP: Could not parse years: " + column, pe);
+ log.warn("SLLSP: Could not parse years: " + column, pe);
}
return null;
}
- private ImportGrainFraction buildGrainFraction(String gfStr) {
- Matcher a = META_GRAIN_FRACTION_A.matcher(gfStr);
- if (a.matches()) {
- String lowerA = a.group(2);
- String lowerB = a.group(3);
-
- String upperA = a.group(4);
- String upperB = a.group(5);
-
- String lower = lowerA != null ? lowerA : lowerB;
- String upper = upperA != null ? upperA : upperB;
-
- try {
- return new ImportGrainFraction(
- getGrainFractionTypeName(this.description),
- nf.parse(lower).doubleValue(),
- nf.parse(upper).doubleValue()
- );
- }
- catch (ParseException pe) {
- log.warn("SYP: Could not parse ranges of: '" + gfStr + "'");
- }
- }
-
- Matcher b = META_GRAIN_FRACTION_B.matcher(gfStr);
- if (b.matches()) {
- String lowerA = b.group(4);
- String lowerB = b.group(5);
- String upperA = b.group(6);
- String upperB = b.group(7);
-
- String lower = lowerA != null ? lowerA : lowerB;
- String upper = upperA != null ? upperA : upperB;
-
- try {
- return new ImportGrainFraction(
- getGrainFractionTypeName(this.description),
- nf.parse(lower).doubleValue(),
- nf.parse(upper).doubleValue()
- );
- }
- catch (ParseException pe) {
- log.warn("SYP: Could not parse ranges of: '" + gfStr + "'");
- }
- }
-
- Matcher c = META_GRAIN_FRACTION_C.matcher(gfStr);
- if (c.matches()) {
- String oper = c.group(1);
- String valueStr = c.group(3);
-
- try {
- Double value = nf.parse(valueStr).doubleValue();
-
- if (oper.equals(">")) {
- return new ImportGrainFraction(
- getGrainFractionTypeName(this.description),
- value,
- null
- );
- }
- else {
- return new ImportGrainFraction(
- getGrainFractionTypeName(this.description),
- null,
- value
- );
- }
- }
- catch (ParseException pe) {
- log.warn("SYP: Could not parse ranges of: '" + gfStr + "'");
- }
- }
-
- log.warn("SYP: Unknown grain fraction: '" + gfStr + "'");
- return new ImportGrainFraction(GrainFraction.UNKNOWN);
- }
-
-
- public static String getGrainFractionTypeName(String filename) {
- if (Pattern.matches(FRACTION_COARSE_STR, filename)) {
- return GrainFraction.COARSE;
- }
- else if (Pattern.matches(FRACTION_FINE_MIDDLE_STR, filename)) {
- return GrainFraction.FINE_MIDDLE;
- }
- else if (Pattern.matches(FRACTION_SUSP_SAND_BED, filename) ||
- Pattern.matches(FRACTION_SUSP_SAND_BED_EPOCH, filename)) {
- return GrainFraction.SUSP_SAND_BED;
- }
- else if (Pattern.matches(FRACTION_SUSP_SAND, filename)) {
- return GrainFraction.SUSP_SAND;
- }
- else if (Pattern.matches(FRACTION_SAND, filename)) {
- return GrainFraction.SAND;
- }
- else if (Pattern.matches(FRACTION_SUSPENDED_SEDIMENT, filename)) {
- return GrainFraction.SUSPENDED_SEDIMENT;
- }
- else if (Pattern.matches(FRACTION_TOTAL, filename)) {
- return GrainFraction.TOTAL;
- }
- else {
- log.warn("SYP: Unknown grain fraction '" + filename + "'");
- return GrainFraction.UNKNOWN;
- }
- }
-
-
- public List<ImportSedimentLoadLS> getSedimentYields() {
- return sedimentYields;
+ public List<ImportSedimentLoadLS> getSedimentLoadLSs() {
+ return sedimentLoadLSs;
}
}
// vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/model/SedimentLoadLS.java
--- a/backend/src/main/java/org/dive4elements/river/model/SedimentLoadLS.java Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/model/SedimentLoadLS.java Mon Jul 14 15:36:44 2014 +0200
@@ -24,7 +24,7 @@
import javax.persistence.OneToOne;
-/** SedimentYield of a certain Fraction with possibly many values. */
+/** SedimentLoadLS of a certain Fraction with possibly many values. */
@Entity
@Table(name = "sediment_yield")
public class SedimentLoadLS
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/model/SedimentLoadLSValue.java
--- a/backend/src/main/java/org/dive4elements/river/model/SedimentLoadLSValue.java Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/model/SedimentLoadLSValue.java Mon Jul 14 15:36:44 2014 +0200
@@ -32,23 +32,21 @@
private Integer id;
- private SedimentLoadLS sedimentYield;
+ private SedimentLoadLS sedimentLoadLS;
private Double station;
private Double value;
- private Unit unit;
-
public SedimentLoadLSValue() {
}
public SedimentLoadLSValue(
- SedimentLoadLS sedimentYield,
+ SedimentLoadLS sedimentLoadLS,
Double station,
Double value
) {
- this.sedimentYield = sedimentYield;
+ this.sedimentLoadLS = sedimentLoadLS;
this.station = station;
this.value = value;
}
@@ -72,12 +70,12 @@
@OneToOne
@JoinColumn(name = "sediment_yield_id" )
- public SedimentLoadLS getSedimentYield() {
- return sedimentYield;
+ public SedimentLoadLS getSedimentLoadLS() {
+ return sedimentLoadLS;
}
- public void setSedimentYield(SedimentLoadLS sedimentYield) {
- this.sedimentYield = sedimentYield;
+ public void setSedimentLoadLS(SedimentLoadLS sedimentLoadLS) {
+ this.sedimentLoadLS = sedimentLoadLS;
}
@Column(name="station")
More information about the Dive4Elements-commits
mailing list