[PATCH 3 of 4] Checks added for missing import directory

Wald Commits scm-commit at wald.intevation.org
Mon Apr 9 09:08:50 CEST 2018


# HG changeset patch
# User mschaefer
# Date 1523203772 -7200
# Node ID ae76f618d9902d97595ba6065b2fb9e851212a57
# Parent  5ff8ce9a2e0612a19e070853d4156075d32820f5
Checks added for missing import directory

diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/ImportRiver.java
--- a/backend/src/main/java/org/dive4elements/river/importer/ImportRiver.java	Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/ImportRiver.java	Sun Apr 08 18:09:32 2018 +0200
@@ -391,6 +391,10 @@
         return this.wstFile.getParentFile().getParentFile().getParentFile();
     }
 
+    public AnnotationClassifier getAnnotationClassifier() {
+        return this.annotationClassifier;
+    }
+
     public void parseDependencies() throws IOException {
         log.info("Root dir is '" + getRiverDir() + "'");
         parseGauges();
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/common/AbstractKmLineImport.java
--- a/backend/src/main/java/org/dive4elements/river/importer/common/AbstractKmLineImport.java	Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/common/AbstractKmLineImport.java	Sun Apr 08 18:09:32 2018 +0200
@@ -29,10 +29,12 @@
 
     protected StoreMode storeMode;
 
+
     /***** CONSTRUCTOR *****/
 
     public AbstractKmLineImport(final double km) {
         this.station = km;
+        this.storeMode = StoreMode.NONE;
     }
 
 
@@ -52,10 +54,8 @@
      * it
      */
     protected KMTUPLE getPeer(final SERIES parent, final StoreMode parentStoreMode) {
-        if (this.peer != null) {
-            this.storeMode = StoreMode.NONE;
+        if (this.peer != null)
             return this.peer;
-        }
         final Session session = ImporterSession.getInstance().getDatabaseSession();
         KMTUPLE value = null;
         if (parentStoreMode != StoreMode.INSERT) {
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/common/AbstractSeriesImport.java
--- a/backend/src/main/java/org/dive4elements/river/importer/common/AbstractSeriesImport.java	Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/common/AbstractSeriesImport.java	Sun Apr 08 18:09:32 2018 +0200
@@ -45,7 +45,7 @@
 
     protected Map<StoreMode, Integer> valueStoreCount;
 
-    protected StoreMode seriesStoreMode;
+    protected StoreMode storeMode;
 
 
     /***** CONSTRUCTORS *****/
@@ -55,6 +55,7 @@
         this.valueStoreCount = new EnumMap<>(StoreMode.class);
         for (final StoreMode mode : StoreMode.values())
             this.valueStoreCount.put(mode, Integer.valueOf(0));
+        this.storeMode = StoreMode.NONE;
     }
 
     public AbstractSeriesImport(final String filename) {
@@ -118,11 +119,11 @@
         final SERIES peer = getPeer(river);
         if (peer != null) {
             for (final KMLINE value : this.values) {
-                incrementValueStoreCount(value.store(peer, this.seriesStoreMode));
+                incrementValueStoreCount(value.store(peer, this.storeMode));
             }
         }
         ImporterSession.getInstance().getDatabaseSession().flush();
-        return this.seriesStoreMode;
+        return this.storeMode;
     }
 
     /**
@@ -139,21 +140,19 @@
     /**
      * Gets the model object of the data series, inserting it into the database if not already existing
      */
-    public SERIES getPeer(final River river) {
-        if (this.peer != null) {
-            this.seriesStoreMode = StoreMode.NONE;
+    protected SERIES getPeer(final River river) {
+        if (this.peer != null)
             return this.peer;
-        }
         final Session session = ImporterSession.getInstance().getDatabaseSession();
         final List<SERIES> rows = querySeriesItem(session, river);
         if (rows.isEmpty()) {
             getLog().info("Create new database instance");
             this.peer = createSeriesItem(river);
             session.save(this.peer);
-            this.seriesStoreMode = StoreMode.INSERT;
+            this.storeMode = StoreMode.INSERT;
         } else {
             this.peer = rows.get(0);
-            this.seriesStoreMode = StoreMode.UPDATE;
+            this.storeMode = StoreMode.UPDATE;
         }
         return this.peer;
     }
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/BedMobilityParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/BedMobilityParser.java	Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/BedMobilityParser.java	Sun Apr 08 18:09:32 2018 +0200
@@ -66,7 +66,9 @@
      */
     public static List<BedMobilityParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
         final List<BedMobilityParser> parsers = new ArrayList<>();
-        parsers.add(new BedMobilityParser(new File(importDir, IMPORT_FILENAME), new File(relativeDir, IMPORT_FILENAME), river));
+        final File importFile = new File(importDir, IMPORT_FILENAME);
+        if (importFile.exists())
+            parsers.add(new BedMobilityParser(importFile, new File(relativeDir, IMPORT_FILENAME), river));
         return parsers;
     }
 
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/ChannelParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/ChannelParser.java	Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/ChannelParser.java	Sun Apr 08 18:09:32 2018 +0200
@@ -79,7 +79,9 @@
      */
     public static List<ChannelParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
         final List<ChannelParser> parsers = new ArrayList<>();
-        parsers.add(new ChannelParser(new File(importDir, IMPORT_FILENAME), new File(relativeDir, IMPORT_FILENAME), river));
+        final File importFile = new File(importDir, IMPORT_FILENAME);
+        if (importFile.exists())
+            parsers.add(new ChannelParser(importFile, new File(relativeDir, IMPORT_FILENAME), river));
         return parsers;
     }
 
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/CollisionParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/CollisionParser.java	Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/CollisionParser.java	Sun Apr 08 18:09:32 2018 +0200
@@ -100,8 +100,10 @@
      */
     public static List<CollisionParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
         final List<CollisionParser> parsers = new ArrayList<>();
-        for (final File file : listFiles(importDir, ".csv"))
-            parsers.add(new CollisionParser(file, new File(relativeDir, file.getName()), river));
+        if (importDir.exists()) {
+            for (final File file : listFiles(importDir, ".csv"))
+                parsers.add(new CollisionParser(file, new File(relativeDir, file.getName()), river));
+        }
         return parsers;
     }
 
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DailyDischargeParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DailyDischargeParser.java	Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DailyDischargeParser.java	Sun Apr 08 18:09:32 2018 +0200
@@ -79,14 +79,16 @@
      */
     public static List<DailyDischargeParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
         final List<DailyDischargeParser> parsers = new ArrayList<>();
-        final File[] files = importDir.listFiles(new FilenameFilter() {
-            @Override
-            public boolean accept(final File dir, final String name) {
-                return IMPORT_FILENAME.matcher(name).matches();
-            }
-        });
-        for (final File file : files)
-            parsers.add(new DailyDischargeParser(file, new File(relativeDir, file.getName()), river));
+        if (importDir.exists()) {
+            final File[] files = importDir.listFiles(new FilenameFilter() {
+                @Override
+                public boolean accept(final File dir, final String name) {
+                    return IMPORT_FILENAME.matcher(name).matches();
+                }
+            });
+            for (final File file : files)
+                parsers.add(new DailyDischargeParser(file, new File(relativeDir, file.getName()), river));
+        }
         return parsers;
     }
 
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DepthEvolutionParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DepthEvolutionParser.java	Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DepthEvolutionParser.java	Sun Apr 08 18:09:32 2018 +0200
@@ -77,8 +77,9 @@
      */
     public static List<DepthEvolutionParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
         final List<DepthEvolutionParser> parsers = new ArrayList<>();
-        for (final File file : listFiles(importDir, ".csv"))
-            parsers.add(new DepthEvolutionParser(file, new File(relativeDir, file.getName()), river));
+        if (importDir.exists())
+            for (final File file : listFiles(importDir, ".csv"))
+                parsers.add(new DepthEvolutionParser(file, new File(relativeDir, file.getName()), river));
         return parsers;
     }
 
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthParser.java	Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthParser.java	Sun Apr 08 18:09:32 2018 +0200
@@ -87,8 +87,9 @@
      */
     public static List<FlowDepthParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
         final List<FlowDepthParser> parsers = new ArrayList<>();
-        for (final File file : listFiles(importDir, ".csv"))
-            parsers.add(new FlowDepthParser(file, new File(relativeDir, file.getName()), river));
+        if (importDir.exists())
+            for (final File file : listFiles(importDir, ".csv"))
+                parsers.add(new FlowDepthParser(file, new File(relativeDir, file.getName()), river));
         return parsers;
     }
 
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/InfrastructureParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/InfrastructureParser.java	Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/InfrastructureParser.java	Sun Apr 08 18:09:32 2018 +0200
@@ -26,7 +26,6 @@
 import org.dive4elements.river.importer.common.ParsingState;
 import org.dive4elements.river.importer.sinfo.importitem.InfrastructureKmLineImport;
 import org.dive4elements.river.importer.sinfo.importitem.InfrastructureSeriesImport;
-import org.dive4elements.river.model.AnnotationType;
 import org.dive4elements.river.model.sinfo.Infrastructure;
 import org.dive4elements.river.model.sinfo.InfrastructureValue;
 
@@ -66,8 +65,6 @@
 
     private final HashMap<String, ImportAttribute> bankAttributes;
 
-    private final HashMap<String, ImportAnnotationType> types;
-
 
     /***** CONSTRUCTORS *****/
 
@@ -79,9 +76,6 @@
         this.bankAttributes.put("links", new ImportAttribute(DB_BANK_LEFT));
         this.bankAttributes.put("rechts", new ImportAttribute(DB_BANK_RIGHT));
         this.bankAttributes.put("", new ImportAttribute(DB_BANK_NULL));
-        this.types = new HashMap<>();
-        for (final AnnotationType type : AnnotationType.getTypes())
-            this.types.put(type.getName().trim().toLowerCase(), new ImportAnnotationType(type.getName()));
     }
 
 
@@ -104,8 +98,9 @@
      */
     public static List<InfrastructureParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
         final List<InfrastructureParser> parsers = new ArrayList<>();
-        for (final File file : listFiles(importDir, ".csv"))
-            parsers.add(new InfrastructureParser(file, new File(relativeDir, file.getName()), river));
+        if (importDir.exists())
+            for (final File file : listFiles(importDir, ".csv"))
+                parsers.add(new InfrastructureParser(file, new File(relativeDir, file.getName()), river));
         return parsers;
     }
 
@@ -137,12 +132,15 @@
         final Matcher m = META_TYPE.matcher(this.currentLine);
         if (m.matches()) {
             this.metaPatternsMatched.add(META_TYPE);
-            if (this.types.containsKey(m.group(1).trim().toLowerCase()))
-                this.seriesHeader.setType(this.types.get(m.group(1).trim().toLowerCase()));
+            if (this.river.getAnnotationClassifier() != null) {
+                final ImportAnnotationType type = this.river.getAnnotationClassifier().classifyDescription(m.group(1).trim(),
+                        this.river.getAnnotationClassifier().getDefaultType());
+                this.seriesHeader.setType(type);
+                log.info(String.format("Type name in file: '%s', will be assigned to database name '%s'", m.group(1).trim(), type.getName()));
+            }
             else {
-                final ImportAnnotationType type = new ImportAnnotationType(m.group(1).trim());
-                this.types.put(m.group(1).trim().toLowerCase(), type);
-                this.seriesHeader.setType(type);
+                log.error("No annotation types file configured, cannot process type '" + m.group(1).trim() + "'");
+                this.headerParsingState = ParsingState.STOP;
             }
             return true;
         }
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/SelectedAdditionalParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/SelectedAdditionalParser.java	Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/SelectedAdditionalParser.java	Sun Apr 08 18:09:32 2018 +0200
@@ -17,6 +17,7 @@
 import java.io.LineNumberReader;
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 
 import org.apache.log4j.Logger;
@@ -43,9 +44,7 @@
 
     private static final Logger log = Logger.getLogger(SelectedAdditionalParser.class);
 
-    private static final String IMPORT_Q_FILENAME = "Mit_Abflussdaten.txt";
-
-    private static final String IMPORT_W_FILENAME = "Ohne_Abflussdaten.txt";
+    private static final String IMPORT_FILENAME = "Zus_Laengsschnitte.txt";
 
     private enum SelectionType {
         WITH_Q("Q", "with discharge"), //
@@ -66,6 +65,13 @@
         public String getLogText() {
             return this.logText;
         }
+
+        public static SelectionType parse(final String path) {
+            if (path.toLowerCase().endsWith(".wst"))
+                return WITH_Q;
+            else
+                return WITHOUT_Q;
+        }
     }
 
     private final File importPath;
@@ -74,19 +80,16 @@
 
     private final ImportRiver river;
 
-    private final SelectionType selectionType;
-
-    private final List<String> links;
+    private final HashMap<String, SelectionType> links;
 
 
     /***** CONSTRUCTORS *****/
 
-    public SelectedAdditionalParser(final File importPath, final File rootRelativePath, final ImportRiver river, final SelectionType selectionType) {
+    public SelectedAdditionalParser(final File importPath, final File rootRelativePath, final ImportRiver river) {
         this.importPath = importPath;
         this.rootRelativePath = rootRelativePath;
         this.river = river;
-        this.selectionType = selectionType;
-        this.links = new ArrayList<>();
+        this.links = new HashMap<>();
     }
 
 
@@ -104,10 +107,9 @@
      */
     public static List<SelectedAdditionalParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
         final List<SelectedAdditionalParser> parsers = new ArrayList<>();
-        parsers.add(new SelectedAdditionalParser(new File(importDir, IMPORT_Q_FILENAME), new File(relativeDir, IMPORT_Q_FILENAME),
-                river, SelectionType.WITH_Q));
-        parsers.add(new SelectedAdditionalParser(new File(importDir, IMPORT_W_FILENAME), new File(relativeDir, IMPORT_W_FILENAME),
-                river, SelectionType.WITHOUT_Q));
+        final File importFile = new File(importDir, IMPORT_FILENAME);
+        if (importFile.exists())
+            parsers.add(new SelectedAdditionalParser(importFile, new File(relativeDir, IMPORT_FILENAME), river));
         return parsers;
     }
 
@@ -123,8 +125,8 @@
                 line = in.readLine();
                 if (line == null)
                     break;
-                if (!line.trim().isEmpty() && !line.trim().startsWith(AbstractParser.START_META_CHAR))
-                    this.links.add(line.trim());
+                if (!line.trim().isEmpty() && !line.trim().startsWith(AbstractParser.START_META_CHAR) && !this.links.containsKey(line.trim()))
+                    this.links.put(line.trim(), SelectionType.parse(line.trim()));
             }
             log.info("Number of file links found: " + this.links.size());
         }
@@ -138,17 +140,16 @@
     public void store() {
         final Session session = ImporterSession.getInstance().getDatabaseSession();
         final SQLQuery reset = session.createSQLQuery("UPDATE wsts SET sinfo_selection = NULL WHERE (river_id=:river_id) AND (kind=1)"
-                + " AND (sinfo_selection=:seltype)");
+                + " AND (sinfo_selection IS NOT NULL)");
         reset.setParameter("river_id", this.river.getPeer().getId());
-        reset.setParameter("seltype", this.selectionType.getKey());
         reset.executeUpdate();
         final Query query = session.createQuery("FROM Wst WHERE (river=:river) AND (kind=1) AND (lower(description) LIKE :path)");
         query.setParameter("river", this.river.getPeer());
         int count = 0;
-        for (final String wstfile : this.links) {
-            count += updateWst(session, query, this.river.getPeer(), wstfile, this.selectionType);
+        for (final String wstfile : this.links.keySet()) {
+            count += updateWst(session, query, this.river.getPeer(), wstfile, this.links.get(wstfile));
         }
-        log.info("Updated " + count + " wsts for selected additionals " + this.selectionType.getLogText());
+        log.info("Updated " + count + " wsts for selected additionals");
     }
 
     private int updateWst(final Session session, final Query query, final River river, final String path, final SelectionType selectionType) {
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhParser.java	Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhParser.java	Sun Apr 08 18:09:32 2018 +0200
@@ -87,8 +87,9 @@
      */
     public static List<TkhParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
         final List<TkhParser> parsers = new ArrayList<>();
-        for (final File file : listFiles(importDir, ".csv"))
-            parsers.add(new TkhParser(file, new File(relativeDir, file.getName()), river));
+        if (importDir.exists())
+            for (final File file : listFiles(importDir, ".csv"))
+                parsers.add(new TkhParser(file, new File(relativeDir, file.getName()), river));
         return parsers;
     }
 
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/uinfo/parsers/SalixParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/uinfo/parsers/SalixParser.java	Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/uinfo/parsers/SalixParser.java	Sun Apr 08 18:09:32 2018 +0200
@@ -87,7 +87,9 @@
      */
     public static List<SalixParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
         final List<SalixParser> parsers = new ArrayList<>();
-        parsers.add(new SalixParser(new File(importDir, IMPORT_FILENAME), new File(relativeDir, IMPORT_FILENAME), river));
+        final File importFile = new File(importDir, IMPORT_FILENAME);
+        if (importFile.exists())
+            parsers.add(new SalixParser(importFile, new File(relativeDir, IMPORT_FILENAME), river));
         return parsers;
     }
 


More information about the Dive4Elements-commits mailing list