[PATCH] SCHEMA CHANGE and Importer (only longitudinal section data so far): only grain fractions given now in schema are valid. We are handling sediment loads, not yields

Wald Commits scm-commit at wald.intevation.org
Mon Jul 14 15:36:51 CEST 2014


# HG changeset patch
# User "Tom Gottfried <tom at intevation.de>"
# Date 1405345004 -7200
# Node ID fd3a24336e6a9f454e2bf21b78decf00119b70cc
# Parent  6954ac9b7591ab2c58b899838c3a73cce6ea6112
SCHEMA CHANGE and Importer (only longitudinal section data so far): only grain fractions given now in schema are valid. We are handling sediment loads, not yields.

diff -r 6954ac9b7591 -r fd3a24336e6a backend/contrib/import_river.sh
--- a/backend/contrib/import_river.sh	Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/contrib/import_river.sh	Mon Jul 14 15:36:44 2014 +0200
@@ -235,7 +235,7 @@
     -Dflys.backend.importer.skip.waterlevels=true \
     -Dflys.backend.importer.skip.sq.relation=true \
     -Dflys.backend.importer.skip.sediment.density=true \
-    -Dflys.backend.importer.skip.sediment.yield=true \
+    -Dflys.backend.importer.skip.sediment.load.ls=true \
     -Dflys.backend.importer.skip.morphological.width=true \
     -Dflys.backend.importer.skip.flow.velocity=true \
     -Dflys.backend.importer.skip.bed.height.single=true \
@@ -282,7 +282,7 @@
     -Dflys.backend.importer.skip.waterlevels=false \
     -Dflys.backend.importer.skip.sq.relation=false \
     -Dflys.backend.importer.skip.sediment.density=false \
-    -Dflys.backend.importer.skip.sediment.yield=false \
+    -Dflys.backend.importer.skip.sediment.load.ls=false \
     -Dflys.backend.importer.skip.morphological.width=false \
     -Dflys.backend.importer.skip.flow.velocity=false \
     -Dflys.backend.importer.skip.bed.height.single=false \
@@ -329,7 +329,7 @@
     -Dflys.backend.importer.skip.waterlevels=true \
     -Dflys.backend.importer.skip.sq.relation=true \
     -Dflys.backend.importer.skip.sediment.density=true \
-    -Dflys.backend.importer.skip.sediment.yield=true \
+    -Dflys.backend.importer.skip.sediment.load.ls=true \
     -Dflys.backend.importer.skip.morphological.width=true \
     -Dflys.backend.importer.skip.flow.velocity=true \
     -Dflys.backend.importer.skip.bed.height.single=true \
diff -r 6954ac9b7591 -r fd3a24336e6a backend/contrib/run_hydr_morph.sh
--- a/backend/contrib/run_hydr_morph.sh	Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/contrib/run_hydr_morph.sh	Mon Jul 14 15:36:44 2014 +0200
@@ -47,7 +47,7 @@
 IMPORTER_SKIP_MORPHOLOGICAL_WIDTH=false
 IMPORTER_SKIP_POROSITY=false
 IMPORTER_SKIP_SEDIMENT_DENSITY=false
-IMPORTER_SKIP_SEDIMENT_YIELD=false
+IMPORTER_SKIP_SEDIMENT_LOAD_LS=false
 IMPORTER_SKIP_SQ_RELATION=false
 IMPORTER_SKIP_WATERLEVELS=false
 IMPORTER_SKIP_WATERLEVEL_DIFFERENCES=false
@@ -95,7 +95,7 @@
     -Dflys.backend.importer.skip.official.lines=$IMPORTER_SKIP_OFFICIAL_LINES \
     -Dflys.backend.importer.skip.prfs=$IMPORTER_SKIP_PRFS \
     -Dflys.backend.importer.skip.sediment.density=$IMPORTER_SKIP_SEDIMENT_DENSITY \
-    -Dflys.backend.importer.skip.sediment.yield=$IMPORTER_SKIP_SEDIMENT_YIELD \
+    -Dflys.backend.importer.skip.sediment.load.ls=$IMPORTER_SKIP_SEDIMENT_LOAD_LS \
     -Dflys.backend.importer.skip.sq.relation=$IMPORTER_SKIP_SQ_RELATION \
     -Dflys.backend.importer.skip.w80s=$IMPORTER_SKIP_W80S \
     -Dflys.backend.importer.skip.w80.csvs=$IMPORTER_SKIP_W80_CSVS \
diff -r 6954ac9b7591 -r fd3a24336e6a backend/doc/documentation/de/importer-hydr-morph.tex
--- a/backend/doc/documentation/de/importer-hydr-morph.tex	Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/doc/documentation/de/importer-hydr-morph.tex	Mon Jul 14 15:36:44 2014 +0200
@@ -211,13 +211,18 @@
 \textit{Morphologie/Geschwindigkeit\_Schubspannung/Modellrechnungen} und\\
 \textit{Morphologie/Geschwindigkeit\_Schubspannung/v-Messungen} geladen.
 
-\subsubsection{Sedimentfracht}
-Der Import der Sedimentfracht kann mit
-\textbf{-Dflys.backend.importer.skip.sediment.yield=true}
-unterdrückt werden. Es werden die CSV-Dateien aus dem Verzeichnis
-\textit{Morphologie/Fracht} geladen. Dabei werden die Dateien aus dem
-Unterverzeichnissen \textit{Einzeljahre} und \textit{Epochen} entsprechend als
-\textit{Einzeljahre} und \textit{Epochen} gespeichert.
+\subsubsection{Sedimentfracht (Längsschnitt-Daten)}
+Mit
+\textbf{-Dflys.backend.importer.skip.sediment.load.ls=true}
+kann der Import der Längsschnitt-Daten (auf freier Strecke)
+der Sedimentfracht unterdrückt werden.
+Es werden die CSV-Dateien aus dem Verzeichnis
+\textit{Morphologie/Fracht/Laengsschnitte} geladen. 
+Dabei werden die Dateien aus dem
+Unterverzeichnissen \textit{Einzeljahre}, \textit{Epochen}
+und \textit{amtliche Epochen} entsprechend als
+\textit{Einzeljahre}, \textit{Epochen} und
+\textit{amtliche Epochen} gespeichert.
 
 \subsubsection{Wasserspiegellagen für M-INFO}
 Mit \textbf{-Dflys.backend.importer.skip.waterlevels=true}
diff -r 6954ac9b7591 -r fd3a24336e6a backend/doc/schema/oracle-minfo.sql
--- a/backend/doc/schema/oracle-minfo.sql	Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/doc/schema/oracle-minfo.sql	Mon Jul 14 15:36:44 2014 +0200
@@ -243,8 +243,21 @@
     name    VARCHAR(64)    NOT NULL,
     lower   NUMBER(38,3),
     upper   NUMBER(38,3),
-    PRIMARY KEY (id)
+    PRIMARY KEY (id),
+    UNIQUE(name, lower, upper)
 );
+-- single fractions
+INSERT INTO grain_fraction VALUES (1, 'coarse', 16, 200);
+INSERT INTO grain_fraction VALUES (2, 'fine_middle', 2, 16);
+INSERT INTO grain_fraction VALUES (3, 'sand', 0.063, 2);
+INSERT INTO grain_fraction VALUES (4, 'susp_sand', 0.063, 2);
+INSERT INTO grain_fraction VALUES (5, 'susp_sand_bed', 0.063, 2);
+INSERT INTO grain_fraction VALUES (6, 'suspended_sediment', 0, 0.063);
+-- aggregations of fractions
+INSERT INTO grain_fraction VALUES (7, 'total', 0, 200);
+INSERT INTO grain_fraction VALUES (8, 'bed_load', 0.063, 200);
+INSERT INTO grain_fraction VALUES (9, 'suspended_load', 0, 2);
+
 
 --lookup table for sediment yield kinds
 CREATE TABLE sediment_yield_kinds (
diff -r 6954ac9b7591 -r fd3a24336e6a backend/doc/schema/postgresql-minfo.sql
--- a/backend/doc/schema/postgresql-minfo.sql	Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/doc/schema/postgresql-minfo.sql	Mon Jul 14 15:36:44 2014 +0200
@@ -243,8 +243,20 @@
     name    VARCHAR(64)    NOT NULL,
     lower   NUMERIC,
     upper   NUMERIC,
-    PRIMARY KEY (id)
+    PRIMARY KEY (id),
+    UNIQUE(name, lower, upper)
 );
+-- single fractions
+INSERT INTO grain_fraction VALUES (1, 'coarse', 16, 200);
+INSERT INTO grain_fraction VALUES (2, 'fine_middle', 2, 16);
+INSERT INTO grain_fraction VALUES (3, 'sand', 0.063, 2);
+INSERT INTO grain_fraction VALUES (4, 'susp_sand', 0.063, 2);
+INSERT INTO grain_fraction VALUES (5, 'susp_sand_bed', 0.063, 2);
+INSERT INTO grain_fraction VALUES (6, 'suspended_sediment', 0, 0.063);
+-- aggregations of fractions
+INSERT INTO grain_fraction VALUES (7, 'total', 0, 200);
+INSERT INTO grain_fraction VALUES (8, 'bed_load', 0.063, 200);
+INSERT INTO grain_fraction VALUES (9, 'suspended_load', 0, 2);
 
 
 --lookup table for sediment yield kinds
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/importer/Config.java
--- a/backend/src/main/java/org/dive4elements/river/importer/Config.java	Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/Config.java	Mon Jul 14 15:36:44 2014 +0200
@@ -85,8 +85,8 @@
     public static final String SKIP_FLOW_VELOCITY =
         "flys.backend.importer.skip.flow.velocity";
 
-    public static final String SKIP_SEDIMENT_YIELD =
-        "flys.backend.importer.skip.sediment.yield";
+    public static final String SKIP_SEDIMENT_LOAD_LS =
+        "flys.backend.importer.skip.sediment.load.ls";
 
     public static final String SKIP_WATERLEVELS =
         "flys.backend.importer.skip.waterlevels";
@@ -227,8 +227,8 @@
         return getFlag(SKIP_FLOW_VELOCITY);
     }
 
-    public boolean skipSedimentYield() {
-        return getFlag(SKIP_SEDIMENT_YIELD);
+    public boolean skipSedimentLoadLS() {
+        return getFlag(SKIP_SEDIMENT_LOAD_LS);
     }
 
     public boolean skipWaterlevels() {
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/importer/ImportGrainFraction.java
--- a/backend/src/main/java/org/dive4elements/river/importer/ImportGrainFraction.java	Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/ImportGrainFraction.java	Mon Jul 14 15:36:44 2014 +0200
@@ -13,24 +13,10 @@
 
 public class ImportGrainFraction {
 
-    private String name;
+    private GrainFraction peer;
 
-    private Double lower;
-    private Double upper;
-
-    public ImportGrainFraction(String name) {
-        this.name = name;
-    }
-
-
-    public ImportGrainFraction(
-        String     name,
-        Double     lower,
-        Double     upper
-    ) {
-        this.name  = name;
-        this.lower = lower;
-        this.upper = upper;
+    public ImportGrainFraction(GrainFraction gf) {
+        this.peer = gf;
     }
 
 
@@ -40,7 +26,7 @@
 
 
     public GrainFraction getPeer() {
-        return ImporterSession.getInstance().getGrainFraction(name);
+        return peer;
     }
 }
 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/importer/ImportRiver.java
--- a/backend/src/main/java/org/dive4elements/river/importer/ImportRiver.java	Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/ImportRiver.java	Mon Jul 14 15:36:44 2014 +0200
@@ -104,13 +104,15 @@
 
     public static final String FLOW_VELOCITY_MEASUREMENTS = "v-Messungen";
 
-    public static final String SEDIMENT_YIELD_DIR = "Fracht";
+    public static final String SEDIMENT_LOAD_DIR = "Fracht";
 
-    public static final String SEDIMENT_YIELD_SINGLE_DIR = "Einzeljahre";
+    public static final String SEDIMENT_LOAD_LS_DIR = "Laengsschnitte";
 
-    public static final String SEDIMENT_YIELD_EPOCH_DIR = "Epochen";
+    public static final String SEDIMENT_LOAD_SINGLE_DIR = "Einzeljahre";
 
-    public static final String SEDIMENT_YIELD_OFF_EPOCH_DIR = "amtliche Epochen";
+    public static final String SEDIMENT_LOAD_EPOCH_DIR = "Epochen";
+
+    public static final String SEDIMENT_LOAD_OFF_EPOCH_DIR = "amtliche Epochen";
 
     public static final String MINFO_FIXATIONS_DIR = "Fixierungsanalyse";
 
@@ -172,7 +174,7 @@
 
     protected List<ImportFlowVelocityMeasurement> flowVelocityMeasurements;
 
-    protected List<ImportSedimentLoadLS> sedimentYields;
+    protected List<ImportSedimentLoadLS> sedimentLoadLSs;
 
     protected List<ImportMeasurementStation> measurementStations;
 
@@ -288,7 +290,7 @@
         morphologicalWidths       = new ArrayList<ImportMorphWidth>();
         flowVelocityModels        = new ArrayList<ImportFlowVelocityModel>();
         flowVelocityMeasurements  = new ArrayList<ImportFlowVelocityMeasurement>();
-        sedimentYields            = new ArrayList<ImportSedimentLoadLS>();
+        sedimentLoadLSs           = new ArrayList<ImportSedimentLoadLS>();
         measurementStations       = new ArrayList<ImportMeasurementStation>();
         sqRelations               = new ArrayList<ImportSQRelation>();
     }
@@ -381,7 +383,7 @@
         parsePorosity();
         parseMorphologicalWidth();
         parseFlowVelocity();
-        parseSedimentYield();
+        parseSedimentLoadLS();
         parseWaterlevels();
         parseWaterlevelDifferences();
         parseMeasurementStations();
@@ -593,7 +595,7 @@
     }
 
 
-    private void parseSedimentYieldDir(
+    private void parseSedimentLoadLSDir(
         File[] files,
         SedimentLoadLSParser parser
     ) throws IOException {
@@ -609,20 +611,25 @@
        }
     }
 
-    protected void parseSedimentYield() throws IOException {
-        if (Config.INSTANCE.skipSedimentYield()) {
-            log.info("skip parsing sediment yield data");
+    protected void parseSedimentLoadLS() throws IOException {
+        if (Config.INSTANCE.skipSedimentLoadLS()) {
+            log.info("skip parsing sediment load longitudinal section data");
             return;
         }
 
-        log.debug("Parse sediment yield data");
+        log.debug("Parse sediment load longitudinal section data");
 
-        File minfoDir         = getMinfoDir();
-        File sedimentYieldDir = new File(minfoDir, SEDIMENT_YIELD_DIR);
+        File minfoDir          = getMinfoDir();
+        File sedimentLoadDir   = new File(minfoDir, SEDIMENT_LOAD_DIR);
+        File sedimentLoadLSDir = new File(sedimentLoadDir,
+                                          SEDIMENT_LOAD_LS_DIR);
 
-        File singleDir   = new File(sedimentYieldDir, SEDIMENT_YIELD_SINGLE_DIR);
-        File epochDir    = new File(sedimentYieldDir, SEDIMENT_YIELD_EPOCH_DIR);
-        File offEpochDir = new File(sedimentYieldDir, SEDIMENT_YIELD_OFF_EPOCH_DIR);
+        File singleDir   = new File(sedimentLoadLSDir,
+                                    SEDIMENT_LOAD_SINGLE_DIR);
+        File epochDir    = new File(sedimentLoadLSDir,
+                                    SEDIMENT_LOAD_EPOCH_DIR);
+        File offEpochDir = new File(sedimentLoadLSDir,
+                                    SEDIMENT_LOAD_OFF_EPOCH_DIR);
 
         File[] singles   = singleDir.listFiles();
         File[] epochs    = epochDir.listFiles();
@@ -634,24 +641,24 @@
             log.warn("Cannot read directory '" + singleDir + "'");
         }
         else {
-            parseSedimentYieldDir(singles, parser);
+            parseSedimentLoadLSDir(singles, parser);
         }
 
         if (epochs == null || epochs.length == 0) {
             log.warn("Cannot read directory '" + epochDir + "'");
         }
         else {
-            parseSedimentYieldDir(epochs, parser);
+            parseSedimentLoadLSDir(epochs, parser);
         }
 
         if (offEpochs == null || offEpochs.length == 0) {
             log.warn("Cannot read directory '" + offEpochDir + "'");
         }
         else {
-            parseSedimentYieldDir(offEpochs, parser);
+            parseSedimentLoadLSDir(offEpochs, parser);
         }
 
-        sedimentYields = parser.getSedimentYields();
+        sedimentLoadLSs = parser.getSedimentLoadLSs();
     }
 
 
@@ -1255,7 +1262,7 @@
         storePorosity();
         storeMorphologicalWidth();
         storeFlowVelocity();
-        storeSedimentYield();
+        storeSedimentLoadLS();
         storeWaterlevels();
         storeWaterlevelDifferences();
         storeMeasurementStations();
@@ -1495,14 +1502,14 @@
     }
 
 
-    public void storeSedimentYield() {
-        if (!Config.INSTANCE.skipSedimentYield()) {
-            log.info("store sediment yield data");
+    public void storeSedimentLoadLS() {
+        if (!Config.INSTANCE.skipSedimentLoadLS()) {
+            log.info("store sediment load longitudinal section data");
 
             River river = getPeer();
 
-            for (ImportSedimentLoadLS sedimentYield: sedimentYields) {
-                sedimentYield.storeDependencies(river);
+            for (ImportSedimentLoadLS sedimentLoadLS: sedimentLoadLSs) {
+                sedimentLoadLS.storeDependencies(river);
             }
         }
     }
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/importer/ImportSedimentLoadLS.java
--- a/backend/src/main/java/org/dive4elements/river/importer/ImportSedimentLoadLS.java	Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/ImportSedimentLoadLS.java	Mon Jul 14 15:36:44 2014 +0200
@@ -83,7 +83,7 @@
                 i++;
             }
 
-            log.info("stored " + i + " sediment yield values.");
+            log.info("stored " + i + " sediment load values.");
         }
     }
 
@@ -98,14 +98,14 @@
         TimeInterval ti = timeInterval != null ? timeInterval.getPeer() : null;
 
         if (ti == null || u == null) {
-            log.warn("Skip invalid SedimentYield: time interval or unit null!");
+            log.warn("Skip invalid SedimentLoadLS: time interval or unit null!");
             return null;
         }
 
         if (peer == null) {
             Session session = ImporterSession.getInstance()
                 .getDatabaseSession();
-            Query query = session.createQuery("from SedimentYield where "
+            Query query = session.createQuery("from SedimentLoadLS where "
                 + "   river=:river and "
                 + "   grainFraction=:grainFraction and " + "   unit=:unit and "
                 + "   timeInterval=:timeInterval and "
@@ -117,16 +117,16 @@
             query.setParameter("timeInterval", ti);
             query.setParameter("description", description);
 
-            List<SedimentLoadLS> yields = query.list();
-            if (yields.isEmpty()) {
-                log.debug("create new SedimentYield");
+            List<SedimentLoadLS> loads = query.list();
+            if (loads.isEmpty()) {
+                log.debug("create new SedimentLoadLS");
 
                 peer = new SedimentLoadLS(river, u, ti, gf, description);
                 peer.setKind(this.kind);
                 session.save(peer);
             }
             else {
-                peer = yields.get(0);
+                peer = loads.get(0);
             }
         }
 
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/importer/ImportSedimentLoadLSValue.java
--- a/backend/src/main/java/org/dive4elements/river/importer/ImportSedimentLoadLSValue.java	Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/ImportSedimentLoadLSValue.java	Mon Jul 14 15:36:44 2014 +0200
@@ -31,28 +31,28 @@
     }
 
 
-    public void storeDependencies(SedimentLoadLS sedimentYield) {
-        getPeer(sedimentYield);
+    public void storeDependencies(SedimentLoadLS sedimentLoadLS) {
+        getPeer(sedimentLoadLS);
     }
 
 
-    public SedimentLoadLSValue getPeer(SedimentLoadLS sedimentYield) {
+    public SedimentLoadLSValue getPeer(SedimentLoadLS sedimentLoadLS) {
         if (peer == null) {
             Session session = ImporterSession.getInstance().getDatabaseSession();
             Query query = session.createQuery(
-                "from SedimentYieldValue where " +
-                "   sedimentYield=:sedimentYield and " +
+                "from SedimentLoadLSValue where " +
+                "   sedimentLoadLS=:sedimentLoadLS and " +
                 "   station=:station and " +
                 "   value=:value"
             );
 
-            query.setParameter("sedimentYield", sedimentYield);
+            query.setParameter("sedimentLoadLS", sedimentLoadLS);
             query.setParameter("station", station);
             query.setParameter("value", value);
 
             List<SedimentLoadLSValue> values = query.list();
             if (values.isEmpty()) {
-                peer = new SedimentLoadLSValue(sedimentYield, station, value);
+                peer = new SedimentLoadLSValue(sedimentLoadLS, station, value);
                 session.save(peer);
             }
             else {
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/importer/parsers/LineParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/parsers/LineParser.java	Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/parsers/LineParser.java	Mon Jul 14 15:36:44 2014 +0200
@@ -113,12 +113,8 @@
     protected static String stripMetaLine(String line) {
         String tmp = line.substring(1, line.length());
 
-        if (tmp.startsWith(" ")) {
-            return tmp.substring(1, tmp.length());
-        }
-        else {
-            return tmp;
-        }
+        // meta-lines often have trailing semicolons in real data
+        return tmp.replaceAll(SEPERATOR_CHAR + "*$", "").trim();
     }
 
     public static double getDouble(String doubleString) throws ParseException {
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/importer/parsers/SedimentLoadLSParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/parsers/SedimentLoadLSParser.java	Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/parsers/SedimentLoadLSParser.java	Mon Jul 14 15:36:44 2014 +0200
@@ -21,16 +21,19 @@
 
 import org.apache.log4j.Logger;
 
+import org.dive4elements.river.importer.ImporterSession;
 import org.dive4elements.river.importer.ImportGrainFraction;
 import org.dive4elements.river.importer.ImportSedimentLoadLS;
 import org.dive4elements.river.importer.ImportSedimentLoadLSValue;
 import org.dive4elements.river.importer.ImportTimeInterval;
 import org.dive4elements.river.importer.ImportUnit;
+
 import org.dive4elements.river.model.GrainFraction;
+
 import org.dive4elements.river.utils.DateUtil;
+import org.dive4elements.river.utils.EpsilonComparator;
 
-
-/** Parses Sediment Yield files. */
+/** Parses sediment load longitudinal section files. */
 public class SedimentLoadLSParser extends LineParser {
 
     private static final Logger log =
@@ -40,33 +43,6 @@
     public static final NumberFormat nf = NumberFormat.getInstance(DEFAULT_LOCALE);
 
 
-    public static final String FRAKTION_START = "Fraktion:";
-
-    public static final String FRACTION_COARSE_STR =
-        ".*Grobkorn.*";
-
-    public static final String FRACTION_FINE_MIDDLE_STR =
-        ".*Fein.Mittel.Kies.*";
-
-    public static final String FRACTION_SAND =
-        ".*Sand.*";
-
-    public static final String FRACTION_SUSP_SAND =
-        ".*susp.Sand.*";
-
-    public static final String FRACTION_SUSP_SAND_BED =
-        ".*bettbild.Anteil.susp.Sand.*";
-
-    public static final String FRACTION_SUSP_SAND_BED_EPOCH =
-        ".*susp.Sand.bettbildAnteil.*";
-
-    public static final String FRACTION_SUSPENDED_SEDIMENT =
-        ".*Schwebstoff.*";
-
-    public static final String FRACTION_TOTAL =
-        ".*gesamt.*";
-
-
     public static final Pattern TIMEINTERVAL_SINGLE =
         Pattern.compile("\\D*([0-9]+?)\\D*");
 
@@ -76,23 +52,20 @@
     public static final Pattern META_FRACTION =
         Pattern.compile("^Fraktion: (.*)");
 
+    public static final Pattern META_FRACTION_NAME =
+        Pattern.compile("^Fraktionsname: (.*)");
+
     public static final Pattern META_UNIT =
         Pattern.compile("^Einheit: \\[(.*)\\].*");
 
     public static final Pattern META_COLUMN_NAMES =
         Pattern.compile("^Fluss-km.*");
 
-    public static final Pattern META_GRAIN_FRACTION_A =
-        Pattern.compile("\\D*(([0-9]+?,[0-9]+?)\\s*-|([0-9]++)\\s*-)(([0-9]+?,[0-9]+?)|([0-9]++))\\s*([a-zA-Z]+?)\\W*\\D*");
+    public static final Pattern META_GRAIN_SIZE =
+        Pattern.compile("([0-9]*,*[0-9]+)-([0-9]*,*[0-9]+) *mm");
 
-    public static final Pattern META_GRAIN_FRACTION_B =
-        Pattern.compile("(<|>){1}\\s*(\\w++)\\s*(([0-9]+?,[0-9]+?)\\s*-|([0-9]++)\\s*-)(([0-9]+?,[0-9]+?)|([0-9]++))\\s*([a-zA-Z]+?)");
 
-    public static final Pattern META_GRAIN_FRACTION_C =
-        Pattern.compile("(<|>){1}\\s*((([0-9]+?,[0-9]+?)|([0-9]++))\\s*(\\w+))");
-
-
-    protected List<ImportSedimentLoadLS> sedimentYields;
+    protected List<ImportSedimentLoadLS> sedimentLoadLSs;
 
     protected ImportSedimentLoadLS[] current;
 
@@ -104,9 +77,13 @@
 
     protected String[] columnNames;
 
+    private String upper;
+
+    private String lower;
+
 
     public SedimentLoadLSParser() {
-        sedimentYields = new ArrayList<ImportSedimentLoadLS>();
+        sedimentLoadLSs = new ArrayList<ImportSedimentLoadLS>();
     }
 
 
@@ -130,7 +107,7 @@
     protected void finish() {
         if (current != null) {
             for (ImportSedimentLoadLS isy: current) {
-                sedimentYields.add(isy);
+                sedimentLoadLSs.add(isy);
             }
         }
 
@@ -156,11 +133,14 @@
         else if (handleMetaFraction(line)) {
             return;
         }
+        else if (handleMetaFractionName(line)) {
+            return;
+        }
         else if (handleColumnNames(line)) {
             return;
         }
         else {
-            log.warn("SYP: Unknown meta line: '" + line + "'");
+            log.warn("SLLSP: Unknown meta line: '" + line + "'");
         }
     }
 
@@ -181,20 +161,60 @@
         Matcher m = META_FRACTION.matcher(line);
 
         if (m.matches()) {
-            String tmp = m.group(1);
+            String interval = m.group(1);
 
-            this.grainFraction = buildGrainFraction(tmp);
-
-            return true;
-        }
-        else if (line.startsWith(FRAKTION_START)) {
-            String newLine = line.replace(FRAKTION_START, "").trim();
-            if (newLine.length() == 0) {
-                log.debug("Found total grain fraction.");
-                this.grainFraction = new ImportGrainFraction(GrainFraction.UNKNOWN);
+            Matcher sizes = META_GRAIN_SIZE.matcher(interval);
+            if (sizes.matches()) {
+                lower = sizes.group(1);
+                upper = sizes.group(2);
 
                 return true;
             }
+
+            log.warn("SLLSP: Unrecognized grain-size interval. Ignored.");
+            return true;
+
+        }
+
+        return false;
+    }
+
+
+    public boolean handleMetaFractionName(String line) {
+        Matcher m = META_FRACTION_NAME.matcher(line);
+
+        if (m.matches()) {
+            String name = m.group(1);
+
+
+            GrainFraction gf = ImporterSession.getInstance().getGrainFraction(name);
+
+            if (gf != null) {
+
+                if (lower != null && upper != null) {
+                    // Validate grain size interval
+                    try {
+                        Double lowval = nf.parse(lower).doubleValue();
+                        Double upval = nf.parse(upper).doubleValue();
+
+                        if (EpsilonComparator.CMP.compare(lowval,
+                                gf.getLower()) != 0 ||
+                            EpsilonComparator.CMP.compare(upval,
+                                gf.getUpper()) != 0) {
+                            log.warn("SLLSP: Invalid grain size for grain fraction '" +
+                                     name + "'. Ignored.");
+                        }
+                    }
+                    catch (ParseException pe) {
+                        log.warn("SLLSP: Could not parse grain-size interval. Ignored.");
+                    }
+                }
+
+                grainFraction = new ImportGrainFraction(gf);
+                return true;
+            }
+
+            log.error("SLLSP: Unknown grain fraction: '" + name + "'");
         }
 
         return false;
@@ -207,7 +227,13 @@
         if (m.matches()) {
             columnNames = line.split(SEPERATOR_CHAR);
 
-            initializeSedimentYields();
+            // 'Fluss-km', 'Hinweise' and at least one data column required
+            if (columnNames.length < 3) {
+                log.error("SLLSP: missing columns.");
+                return true;
+            }
+
+            initializeSedimentLoadLSs();
 
             return true;
         }
@@ -220,7 +246,7 @@
         String[] vals = line.split(SEPERATOR_CHAR);
 
         if (vals == null || vals.length < columnNames.length-1) {
-            log.warn("SYP: skip invalid data line: '" + line + "'");
+            log.warn("SLLSP: skip invalid data line: '" + line + "'");
             return;
         }
 
@@ -238,14 +264,14 @@
             }
         }
         catch (ParseException pe) {
-            log.warn("SYP: unparseable number in data row '" + line + "':", pe);
+            log.warn("SLLSP: unparseable number in data row '" + line + "':", pe);
         }
     }
 
 
-    /** Initialize SedimentYields from columns, set the kind
+    /** Initialize SedimentLoadLSs from columns, set the kind
      * with respect to file location (offical epoch or not?) */
-    private void initializeSedimentYields() {
+    private void initializeSedimentLoadLSs() {
         // skip first column (Fluss-km) and last column (Hinweise)
         current = new ImportSedimentLoadLS[columnNames.length-2];
 
@@ -288,127 +314,18 @@
                 return new ImportTimeInterval(DateUtil.getStartDateFromYear(year));
             }
 
-            log.warn("SYP: Unknown time interval string: '" + column + "'");
+            log.warn("SLLSP: Unknown time interval string: '" + column + "'");
         }
         catch (ParseException pe) {
-            log.warn("SYP: Could not parse years: " + column, pe);
+            log.warn("SLLSP: Could not parse years: " + column, pe);
         }
 
         return null;
     }
 
 
-    private ImportGrainFraction buildGrainFraction(String gfStr) {
-        Matcher a = META_GRAIN_FRACTION_A.matcher(gfStr);
-        if (a.matches()) {
-            String lowerA = a.group(2);
-            String lowerB = a.group(3);
-
-            String upperA = a.group(4);
-            String upperB = a.group(5);
-
-            String lower = lowerA != null ? lowerA : lowerB;
-            String upper = upperA != null ? upperA : upperB;
-
-            try {
-                return new ImportGrainFraction(
-                    getGrainFractionTypeName(this.description),
-                    nf.parse(lower).doubleValue(),
-                    nf.parse(upper).doubleValue()
-                );
-            }
-            catch (ParseException pe) {
-                log.warn("SYP: Could not parse ranges of: '" + gfStr + "'");
-            }
-        }
-
-        Matcher b = META_GRAIN_FRACTION_B.matcher(gfStr);
-        if (b.matches()) {
-            String lowerA  = b.group(4);
-            String lowerB  = b.group(5);
-            String upperA  = b.group(6);
-            String upperB  = b.group(7);
-
-            String lower = lowerA != null ? lowerA : lowerB;
-            String upper = upperA != null ? upperA : upperB;
-
-            try {
-                return new ImportGrainFraction(
-                    getGrainFractionTypeName(this.description),
-                    nf.parse(lower).doubleValue(),
-                    nf.parse(upper).doubleValue()
-                );
-            }
-            catch (ParseException pe) {
-                log.warn("SYP: Could not parse ranges of: '" + gfStr + "'");
-            }
-        }
-
-        Matcher c = META_GRAIN_FRACTION_C.matcher(gfStr);
-        if (c.matches()) {
-            String oper     = c.group(1);
-            String valueStr = c.group(3);
-
-            try {
-                Double value = nf.parse(valueStr).doubleValue();
-
-                if (oper.equals(">")) {
-                    return new ImportGrainFraction(
-                        getGrainFractionTypeName(this.description),
-                        value,
-                        null
-                    );
-                }
-                else {
-                    return new ImportGrainFraction(
-                        getGrainFractionTypeName(this.description),
-                        null,
-                        value
-                    );
-                }
-            }
-            catch (ParseException pe) {
-                log.warn("SYP: Could not parse ranges of: '" + gfStr + "'");
-            }
-        }
-
-        log.warn("SYP: Unknown grain fraction: '" + gfStr + "'");
-        return new ImportGrainFraction(GrainFraction.UNKNOWN);
-    }
-
-
-    public static String getGrainFractionTypeName(String filename) {
-        if (Pattern.matches(FRACTION_COARSE_STR, filename)) {
-            return GrainFraction.COARSE;
-        }
-        else if (Pattern.matches(FRACTION_FINE_MIDDLE_STR, filename)) {
-            return GrainFraction.FINE_MIDDLE;
-        }
-        else if (Pattern.matches(FRACTION_SUSP_SAND_BED, filename) ||
-            Pattern.matches(FRACTION_SUSP_SAND_BED_EPOCH, filename)) {
-            return GrainFraction.SUSP_SAND_BED;
-        }
-        else if (Pattern.matches(FRACTION_SUSP_SAND, filename)) {
-            return GrainFraction.SUSP_SAND;
-        }
-        else if (Pattern.matches(FRACTION_SAND, filename)) {
-            return GrainFraction.SAND;
-        }
-        else if (Pattern.matches(FRACTION_SUSPENDED_SEDIMENT, filename)) {
-            return GrainFraction.SUSPENDED_SEDIMENT;
-        }
-        else if (Pattern.matches(FRACTION_TOTAL, filename)) {
-            return GrainFraction.TOTAL;
-        }
-        else {
-            log.warn("SYP: Unknown grain fraction '" + filename + "'");
-            return GrainFraction.UNKNOWN;
-        }
-    }
-
-
-    public List<ImportSedimentLoadLS> getSedimentYields() {
-        return sedimentYields;
+    public List<ImportSedimentLoadLS> getSedimentLoadLSs() {
+        return sedimentLoadLSs;
     }
 }
 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/model/SedimentLoadLS.java
--- a/backend/src/main/java/org/dive4elements/river/model/SedimentLoadLS.java	Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/model/SedimentLoadLS.java	Mon Jul 14 15:36:44 2014 +0200
@@ -24,7 +24,7 @@
 import javax.persistence.OneToOne;
 
 
-/** SedimentYield of a certain Fraction with possibly many values. */
+/** SedimentLoadLS of a certain Fraction with possibly many values. */
 @Entity
 @Table(name = "sediment_yield")
 public class SedimentLoadLS
diff -r 6954ac9b7591 -r fd3a24336e6a backend/src/main/java/org/dive4elements/river/model/SedimentLoadLSValue.java
--- a/backend/src/main/java/org/dive4elements/river/model/SedimentLoadLSValue.java	Fri Jul 11 13:10:05 2014 +0200
+++ b/backend/src/main/java/org/dive4elements/river/model/SedimentLoadLSValue.java	Mon Jul 14 15:36:44 2014 +0200
@@ -32,23 +32,21 @@
 
     private Integer id;
 
-    private SedimentLoadLS sedimentYield;
+    private SedimentLoadLS sedimentLoadLS;
 
     private Double station;
     private Double value;
 
-    private Unit unit;
-
 
     public SedimentLoadLSValue() {
     }
 
     public SedimentLoadLSValue(
-        SedimentLoadLS sedimentYield,
+        SedimentLoadLS sedimentLoadLS,
         Double        station,
         Double        value
     ) {
-        this.sedimentYield = sedimentYield;
+        this.sedimentLoadLS = sedimentLoadLS;
         this.station       = station;
         this.value         = value;
     }
@@ -72,12 +70,12 @@
 
     @OneToOne
     @JoinColumn(name = "sediment_yield_id" )
-    public SedimentLoadLS getSedimentYield() {
-        return sedimentYield;
+    public SedimentLoadLS getSedimentLoadLS() {
+        return sedimentLoadLS;
     }
 
-    public void setSedimentYield(SedimentLoadLS sedimentYield) {
-        this.sedimentYield = sedimentYield;
+    public void setSedimentLoadLS(SedimentLoadLS sedimentLoadLS) {
+        this.sedimentLoadLS = sedimentLoadLS;
     }
 
     @Column(name="station")


More information about the Dive4Elements-commits mailing list