[PATCH 3 of 4] Checks added for missing import directory
Wald Commits
scm-commit at wald.intevation.org
Mon Apr 9 09:08:50 CEST 2018
# HG changeset patch
# User mschaefer
# Date 1523203772 -7200
# Node ID ae76f618d9902d97595ba6065b2fb9e851212a57
# Parent 5ff8ce9a2e0612a19e070853d4156075d32820f5
Checks added for missing import directory
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/ImportRiver.java
--- a/backend/src/main/java/org/dive4elements/river/importer/ImportRiver.java Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/ImportRiver.java Sun Apr 08 18:09:32 2018 +0200
@@ -391,6 +391,10 @@
return this.wstFile.getParentFile().getParentFile().getParentFile();
}
+ public AnnotationClassifier getAnnotationClassifier() {
+ return this.annotationClassifier;
+ }
+
public void parseDependencies() throws IOException {
log.info("Root dir is '" + getRiverDir() + "'");
parseGauges();
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/common/AbstractKmLineImport.java
--- a/backend/src/main/java/org/dive4elements/river/importer/common/AbstractKmLineImport.java Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/common/AbstractKmLineImport.java Sun Apr 08 18:09:32 2018 +0200
@@ -29,10 +29,12 @@
protected StoreMode storeMode;
+
/***** CONSTRUCTOR *****/
public AbstractKmLineImport(final double km) {
this.station = km;
+ this.storeMode = StoreMode.NONE;
}
@@ -52,10 +54,8 @@
* it
*/
protected KMTUPLE getPeer(final SERIES parent, final StoreMode parentStoreMode) {
- if (this.peer != null) {
- this.storeMode = StoreMode.NONE;
+ if (this.peer != null)
return this.peer;
- }
final Session session = ImporterSession.getInstance().getDatabaseSession();
KMTUPLE value = null;
if (parentStoreMode != StoreMode.INSERT) {
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/common/AbstractSeriesImport.java
--- a/backend/src/main/java/org/dive4elements/river/importer/common/AbstractSeriesImport.java Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/common/AbstractSeriesImport.java Sun Apr 08 18:09:32 2018 +0200
@@ -45,7 +45,7 @@
protected Map<StoreMode, Integer> valueStoreCount;
- protected StoreMode seriesStoreMode;
+ protected StoreMode storeMode;
/***** CONSTRUCTORS *****/
@@ -55,6 +55,7 @@
this.valueStoreCount = new EnumMap<>(StoreMode.class);
for (final StoreMode mode : StoreMode.values())
this.valueStoreCount.put(mode, Integer.valueOf(0));
+ this.storeMode = StoreMode.NONE;
}
public AbstractSeriesImport(final String filename) {
@@ -118,11 +119,11 @@
final SERIES peer = getPeer(river);
if (peer != null) {
for (final KMLINE value : this.values) {
- incrementValueStoreCount(value.store(peer, this.seriesStoreMode));
+ incrementValueStoreCount(value.store(peer, this.storeMode));
}
}
ImporterSession.getInstance().getDatabaseSession().flush();
- return this.seriesStoreMode;
+ return this.storeMode;
}
/**
@@ -139,21 +140,19 @@
/**
* Gets the model object of the data series, inserting it into the database if not already existing
*/
- public SERIES getPeer(final River river) {
- if (this.peer != null) {
- this.seriesStoreMode = StoreMode.NONE;
+ protected SERIES getPeer(final River river) {
+ if (this.peer != null)
return this.peer;
- }
final Session session = ImporterSession.getInstance().getDatabaseSession();
final List<SERIES> rows = querySeriesItem(session, river);
if (rows.isEmpty()) {
getLog().info("Create new database instance");
this.peer = createSeriesItem(river);
session.save(this.peer);
- this.seriesStoreMode = StoreMode.INSERT;
+ this.storeMode = StoreMode.INSERT;
} else {
this.peer = rows.get(0);
- this.seriesStoreMode = StoreMode.UPDATE;
+ this.storeMode = StoreMode.UPDATE;
}
return this.peer;
}
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/BedMobilityParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/BedMobilityParser.java Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/BedMobilityParser.java Sun Apr 08 18:09:32 2018 +0200
@@ -66,7 +66,9 @@
*/
public static List<BedMobilityParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
final List<BedMobilityParser> parsers = new ArrayList<>();
- parsers.add(new BedMobilityParser(new File(importDir, IMPORT_FILENAME), new File(relativeDir, IMPORT_FILENAME), river));
+ final File importFile = new File(importDir, IMPORT_FILENAME);
+ if (importFile.exists())
+ parsers.add(new BedMobilityParser(importFile, new File(relativeDir, IMPORT_FILENAME), river));
return parsers;
}
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/ChannelParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/ChannelParser.java Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/ChannelParser.java Sun Apr 08 18:09:32 2018 +0200
@@ -79,7 +79,9 @@
*/
public static List<ChannelParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
final List<ChannelParser> parsers = new ArrayList<>();
- parsers.add(new ChannelParser(new File(importDir, IMPORT_FILENAME), new File(relativeDir, IMPORT_FILENAME), river));
+ final File importFile = new File(importDir, IMPORT_FILENAME);
+ if (importFile.exists())
+ parsers.add(new ChannelParser(importFile, new File(relativeDir, IMPORT_FILENAME), river));
return parsers;
}
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/CollisionParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/CollisionParser.java Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/CollisionParser.java Sun Apr 08 18:09:32 2018 +0200
@@ -100,8 +100,10 @@
*/
public static List<CollisionParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
final List<CollisionParser> parsers = new ArrayList<>();
- for (final File file : listFiles(importDir, ".csv"))
- parsers.add(new CollisionParser(file, new File(relativeDir, file.getName()), river));
+ if (importDir.exists()) {
+ for (final File file : listFiles(importDir, ".csv"))
+ parsers.add(new CollisionParser(file, new File(relativeDir, file.getName()), river));
+ }
return parsers;
}
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DailyDischargeParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DailyDischargeParser.java Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DailyDischargeParser.java Sun Apr 08 18:09:32 2018 +0200
@@ -79,14 +79,16 @@
*/
public static List<DailyDischargeParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
final List<DailyDischargeParser> parsers = new ArrayList<>();
- final File[] files = importDir.listFiles(new FilenameFilter() {
- @Override
- public boolean accept(final File dir, final String name) {
- return IMPORT_FILENAME.matcher(name).matches();
- }
- });
- for (final File file : files)
- parsers.add(new DailyDischargeParser(file, new File(relativeDir, file.getName()), river));
+ if (importDir.exists()) {
+ final File[] files = importDir.listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(final File dir, final String name) {
+ return IMPORT_FILENAME.matcher(name).matches();
+ }
+ });
+ for (final File file : files)
+ parsers.add(new DailyDischargeParser(file, new File(relativeDir, file.getName()), river));
+ }
return parsers;
}
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DepthEvolutionParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DepthEvolutionParser.java Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/DepthEvolutionParser.java Sun Apr 08 18:09:32 2018 +0200
@@ -77,8 +77,9 @@
*/
public static List<DepthEvolutionParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
final List<DepthEvolutionParser> parsers = new ArrayList<>();
- for (final File file : listFiles(importDir, ".csv"))
- parsers.add(new DepthEvolutionParser(file, new File(relativeDir, file.getName()), river));
+ if (importDir.exists())
+ for (final File file : listFiles(importDir, ".csv"))
+ parsers.add(new DepthEvolutionParser(file, new File(relativeDir, file.getName()), river));
return parsers;
}
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthParser.java Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/FlowDepthParser.java Sun Apr 08 18:09:32 2018 +0200
@@ -87,8 +87,9 @@
*/
public static List<FlowDepthParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
final List<FlowDepthParser> parsers = new ArrayList<>();
- for (final File file : listFiles(importDir, ".csv"))
- parsers.add(new FlowDepthParser(file, new File(relativeDir, file.getName()), river));
+ if (importDir.exists())
+ for (final File file : listFiles(importDir, ".csv"))
+ parsers.add(new FlowDepthParser(file, new File(relativeDir, file.getName()), river));
return parsers;
}
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/InfrastructureParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/InfrastructureParser.java Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/InfrastructureParser.java Sun Apr 08 18:09:32 2018 +0200
@@ -26,7 +26,6 @@
import org.dive4elements.river.importer.common.ParsingState;
import org.dive4elements.river.importer.sinfo.importitem.InfrastructureKmLineImport;
import org.dive4elements.river.importer.sinfo.importitem.InfrastructureSeriesImport;
-import org.dive4elements.river.model.AnnotationType;
import org.dive4elements.river.model.sinfo.Infrastructure;
import org.dive4elements.river.model.sinfo.InfrastructureValue;
@@ -66,8 +65,6 @@
private final HashMap<String, ImportAttribute> bankAttributes;
- private final HashMap<String, ImportAnnotationType> types;
-
/***** CONSTRUCTORS *****/
@@ -79,9 +76,6 @@
this.bankAttributes.put("links", new ImportAttribute(DB_BANK_LEFT));
this.bankAttributes.put("rechts", new ImportAttribute(DB_BANK_RIGHT));
this.bankAttributes.put("", new ImportAttribute(DB_BANK_NULL));
- this.types = new HashMap<>();
- for (final AnnotationType type : AnnotationType.getTypes())
- this.types.put(type.getName().trim().toLowerCase(), new ImportAnnotationType(type.getName()));
}
@@ -104,8 +98,9 @@
*/
public static List<InfrastructureParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
final List<InfrastructureParser> parsers = new ArrayList<>();
- for (final File file : listFiles(importDir, ".csv"))
- parsers.add(new InfrastructureParser(file, new File(relativeDir, file.getName()), river));
+ if (importDir.exists())
+ for (final File file : listFiles(importDir, ".csv"))
+ parsers.add(new InfrastructureParser(file, new File(relativeDir, file.getName()), river));
return parsers;
}
@@ -137,12 +132,15 @@
final Matcher m = META_TYPE.matcher(this.currentLine);
if (m.matches()) {
this.metaPatternsMatched.add(META_TYPE);
- if (this.types.containsKey(m.group(1).trim().toLowerCase()))
- this.seriesHeader.setType(this.types.get(m.group(1).trim().toLowerCase()));
+ if (this.river.getAnnotationClassifier() != null) {
+ final ImportAnnotationType type = this.river.getAnnotationClassifier().classifyDescription(m.group(1).trim(),
+ this.river.getAnnotationClassifier().getDefaultType());
+ this.seriesHeader.setType(type);
+ log.info(String.format("Type name in file: '%s', will be assigned to database name '%s'", m.group(1).trim(), type.getName()));
+ }
else {
- final ImportAnnotationType type = new ImportAnnotationType(m.group(1).trim());
- this.types.put(m.group(1).trim().toLowerCase(), type);
- this.seriesHeader.setType(type);
+ log.error("No annotation types file configured, cannot process type '" + m.group(1).trim() + "'");
+ this.headerParsingState = ParsingState.STOP;
}
return true;
}
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/SelectedAdditionalParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/SelectedAdditionalParser.java Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/SelectedAdditionalParser.java Sun Apr 08 18:09:32 2018 +0200
@@ -17,6 +17,7 @@
import java.io.LineNumberReader;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import org.apache.log4j.Logger;
@@ -43,9 +44,7 @@
private static final Logger log = Logger.getLogger(SelectedAdditionalParser.class);
- private static final String IMPORT_Q_FILENAME = "Mit_Abflussdaten.txt";
-
- private static final String IMPORT_W_FILENAME = "Ohne_Abflussdaten.txt";
+ private static final String IMPORT_FILENAME = "Zus_Laengsschnitte.txt";
private enum SelectionType {
WITH_Q("Q", "with discharge"), //
@@ -66,6 +65,13 @@
public String getLogText() {
return this.logText;
}
+
+ public static SelectionType parse(final String path) {
+ if (path.toLowerCase().endsWith(".wst"))
+ return WITH_Q;
+ else
+ return WITHOUT_Q;
+ }
}
private final File importPath;
@@ -74,19 +80,16 @@
private final ImportRiver river;
- private final SelectionType selectionType;
-
- private final List<String> links;
+ private final HashMap<String, SelectionType> links;
/***** CONSTRUCTORS *****/
- public SelectedAdditionalParser(final File importPath, final File rootRelativePath, final ImportRiver river, final SelectionType selectionType) {
+ public SelectedAdditionalParser(final File importPath, final File rootRelativePath, final ImportRiver river) {
this.importPath = importPath;
this.rootRelativePath = rootRelativePath;
this.river = river;
- this.selectionType = selectionType;
- this.links = new ArrayList<>();
+ this.links = new HashMap<>();
}
@@ -104,10 +107,9 @@
*/
public static List<SelectedAdditionalParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
final List<SelectedAdditionalParser> parsers = new ArrayList<>();
- parsers.add(new SelectedAdditionalParser(new File(importDir, IMPORT_Q_FILENAME), new File(relativeDir, IMPORT_Q_FILENAME),
- river, SelectionType.WITH_Q));
- parsers.add(new SelectedAdditionalParser(new File(importDir, IMPORT_W_FILENAME), new File(relativeDir, IMPORT_W_FILENAME),
- river, SelectionType.WITHOUT_Q));
+ final File importFile = new File(importDir, IMPORT_FILENAME);
+ if (importFile.exists())
+ parsers.add(new SelectedAdditionalParser(importFile, new File(relativeDir, IMPORT_FILENAME), river));
return parsers;
}
@@ -123,8 +125,8 @@
line = in.readLine();
if (line == null)
break;
- if (!line.trim().isEmpty() && !line.trim().startsWith(AbstractParser.START_META_CHAR))
- this.links.add(line.trim());
+ if (!line.trim().isEmpty() && !line.trim().startsWith(AbstractParser.START_META_CHAR) && !this.links.containsKey(line.trim()))
+ this.links.put(line.trim(), SelectionType.parse(line.trim()));
}
log.info("Number of file links found: " + this.links.size());
}
@@ -138,17 +140,16 @@
public void store() {
final Session session = ImporterSession.getInstance().getDatabaseSession();
final SQLQuery reset = session.createSQLQuery("UPDATE wsts SET sinfo_selection = NULL WHERE (river_id=:river_id) AND (kind=1)"
- + " AND (sinfo_selection=:seltype)");
+ + " AND (sinfo_selection IS NOT NULL)");
reset.setParameter("river_id", this.river.getPeer().getId());
- reset.setParameter("seltype", this.selectionType.getKey());
reset.executeUpdate();
final Query query = session.createQuery("FROM Wst WHERE (river=:river) AND (kind=1) AND (lower(description) LIKE :path)");
query.setParameter("river", this.river.getPeer());
int count = 0;
- for (final String wstfile : this.links) {
- count += updateWst(session, query, this.river.getPeer(), wstfile, this.selectionType);
+ for (final String wstfile : this.links.keySet()) {
+ count += updateWst(session, query, this.river.getPeer(), wstfile, this.links.get(wstfile));
}
- log.info("Updated " + count + " wsts for selected additionals " + this.selectionType.getLogText());
+ log.info("Updated " + count + " wsts for selected additionals");
}
private int updateWst(final Session session, final Query query, final River river, final String path, final SelectionType selectionType) {
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhParser.java Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/sinfo/parsers/TkhParser.java Sun Apr 08 18:09:32 2018 +0200
@@ -87,8 +87,9 @@
*/
public static List<TkhParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
final List<TkhParser> parsers = new ArrayList<>();
- for (final File file : listFiles(importDir, ".csv"))
- parsers.add(new TkhParser(file, new File(relativeDir, file.getName()), river));
+ if (importDir.exists())
+ for (final File file : listFiles(importDir, ".csv"))
+ parsers.add(new TkhParser(file, new File(relativeDir, file.getName()), river));
return parsers;
}
diff -r 5ff8ce9a2e06 -r ae76f618d990 backend/src/main/java/org/dive4elements/river/importer/uinfo/parsers/SalixParser.java
--- a/backend/src/main/java/org/dive4elements/river/importer/uinfo/parsers/SalixParser.java Sun Apr 08 18:08:35 2018 +0200
+++ b/backend/src/main/java/org/dive4elements/river/importer/uinfo/parsers/SalixParser.java Sun Apr 08 18:09:32 2018 +0200
@@ -87,7 +87,9 @@
*/
public static List<SalixParser> createParsers(final File importDir, final File relativeDir, final ImportRiver river) {
final List<SalixParser> parsers = new ArrayList<>();
- parsers.add(new SalixParser(new File(importDir, IMPORT_FILENAME), new File(relativeDir, IMPORT_FILENAME), river));
+ final File importFile = new File(importDir, IMPORT_FILENAME);
+ if (importFile.exists())
+ parsers.add(new SalixParser(importFile, new File(relativeDir, IMPORT_FILENAME), river));
return parsers;
}
More information about the Dive4Elements-commits
mailing list