From 34ba7ebceb4f6b3dd07fc4965ba0e10c73f8a7d5 Mon Sep 17 00:00:00 2001 From: Douglas Gillespie <50671166+douggillespie@users.noreply.github.com> Date: Fri, 23 Aug 2024 09:15:40 +0100 Subject: [PATCH] Merge from DG (#150) * Localization output * update localiser output * Start effort management system * Start of Effort plotting Strat of effort plotting on map. Framework for using Effort data in other areas (such as Tethys output). * Logger forms update Effort and Symbol selectors working with Logger forms. Also functions to add, edit and delete form rows in Viewer mode. * Update LoggerFormGraphics.java add in correct symbol managemet to forms graphics. * Effort lines on map Sort of working OK in real time mode. * Working effort system Currently only for map, but seems to work OK * Update Tethys to latest nilus schema * Raven importer Start of a system for a raven importer. Not quite working yet. * Raven import Basic functionality working. Not nice to use though. * Tethys Localization work Abstracting out writing of localization objects and document header information so that individual localisers can give fine scale control of this stuff. * updated Nilus A few updates around track and target motion measures. * FX Plot for raven data Also sorted out symbols a bit and improved symbol selector in Generic plots. * Update spectrogram mark bearing display Remove the 90-angle bit * Raven extra columns Logging of data from additional Raven table columns --- .settings/org.eclipse.jdt.core.prefs | 6 +- src/generalDatabase/EmptyTableDefinition.java | 8 +- src/ravendata/RavenColumnInfo.java | 35 ++++++ src/ravendata/RavenControl.java | 26 +++- src/ravendata/RavenDataRow.java | 56 ++++++--- src/ravendata/RavenDataUnit.java | 28 +++++ src/ravendata/RavenFileReader.java | 113 +++++++++++++++++- src/ravendata/RavenLogging.java | 67 ++++++++++- src/ravendata/RavenParameters.java | 16 +++ src/ravendata/RavenProcess.java | 23 +++- 10 files changed, 339 insertions(+), 39 deletions(-) create mode 100644 src/ravendata/RavenColumnInfo.java diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 7b5e9ffa..86bf912f 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -11,9 +11,9 @@ org.eclipse.jdt.core.codeComplete.staticFinalFieldPrefixes= org.eclipse.jdt.core.codeComplete.staticFinalFieldSuffixes= org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=17 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=19 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=17 +org.eclipse.jdt.core.compiler.compliance=19 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -23,4 +23,4 @@ org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning org.eclipse.jdt.core.compiler.release=enabled -org.eclipse.jdt.core.compiler.source=17 +org.eclipse.jdt.core.compiler.source=19 diff --git a/src/generalDatabase/EmptyTableDefinition.java b/src/generalDatabase/EmptyTableDefinition.java index 53cfd1fb..628c1e8f 100644 --- a/src/generalDatabase/EmptyTableDefinition.java +++ b/src/generalDatabase/EmptyTableDefinition.java @@ -402,10 +402,10 @@ public class EmptyTableDefinition implements Cloneable { protected EmptyTableDefinition clone() { try { EmptyTableDefinition clone = (EmptyTableDefinition) super.clone(); - clone.pamTableItems = new ArrayList<>(); - for (PamTableItem tableItem : this.pamTableItems) { - clone.pamTableItems.add(tableItem.clone()); - } +// clone.pamTableItems = new ArrayList<>(); +// for (PamTableItem tableItem : this.pamTableItems) { +// clone.pamTableItems.add(tableItem.clone()); +// } return clone; } catch (CloneNotSupportedException e) { e.printStackTrace(); diff --git a/src/ravendata/RavenColumnInfo.java b/src/ravendata/RavenColumnInfo.java new file mode 100644 index 00000000..2b8954f4 --- /dev/null +++ b/src/ravendata/RavenColumnInfo.java @@ -0,0 +1,35 @@ +package ravendata; + +import java.io.Serializable; + +/** + * Information on a Raven table non standard column. + * @author dg50 + * + */ +public class RavenColumnInfo implements Serializable { + + private static final long serialVersionUID = 1L; + + public int ravenTableIndex; + + public String name; + + public int maxStrLength; + + public Integer sqlType; + + public RavenColumnInfo(int ravenTableIndex, String name) { + super(); + this.ravenTableIndex = ravenTableIndex; + this.name = name; + } + + public RavenColumnInfo(String name, int maxStrLength, Integer sqlType) { + super(); + this.name = name; + this.maxStrLength = maxStrLength; + this.sqlType = sqlType; + } + +} diff --git a/src/ravendata/RavenControl.java b/src/ravendata/RavenControl.java index 7769770d..69f7f525 100644 --- a/src/ravendata/RavenControl.java +++ b/src/ravendata/RavenControl.java @@ -9,16 +9,11 @@ import java.util.ArrayList; import javax.swing.JMenuItem; -import Acquisition.AcquisitionControl; -import PamController.InputStoreInfo; import PamController.PamConfiguration; import PamController.PamControlledUnit; import PamController.PamControlledUnitSettings; -import PamController.PamController; import PamController.PamSettingManager; import PamController.PamSettings; -import PamView.dialog.warn.WarnOnce; -import pamViewFX.PamSettingsMenuPane; import ravendata.swing.RavenImportDialog; /** @@ -40,6 +35,9 @@ public class RavenControl extends PamControlledUnit implements PamSettings { this.ravenProcess = new RavenProcess(this); addPamProcess(ravenProcess); PamSettingManager.getInstance().registerSettings(this); + if (isViewer()) { + ravenProcess.getRavenLogging().addExtraColumns(ravenParameters.getExtraColumns()); + } } public RavenControl(PamConfiguration pamConfiguration, String unitType, String unitName) { @@ -77,11 +75,29 @@ public class RavenControl extends PamControlledUnit implements PamSettings { e.printStackTrace(); return; } + + sortExtraColumns(fileReader, ravenData); + if (ravenData != null) { ravenProcess.createPAMGuardData(fileReader, ravenData); } } + /** + * Check some information on column types. + * @param fileReader + * @param ravenData + */ + private void sortExtraColumns(RavenFileReader fileReader, ArrayList ravenData) { + /* + * primarily need to add the extra columns to the SQL interface to make sure it's going to + * save the additional data. + * Also put them into the module parameters. + */ + ravenParameters.setExtraColumns(fileReader.getExtraColumns()); + ravenProcess.getRavenLogging().addExtraColumns(fileReader.getExtraColumns()); + } + @Override diff --git a/src/ravendata/RavenDataRow.java b/src/ravendata/RavenDataRow.java index b1f65a13..8924ef9f 100644 --- a/src/ravendata/RavenDataRow.java +++ b/src/ravendata/RavenDataRow.java @@ -1,5 +1,8 @@ package ravendata; +import java.util.ArrayList; +import java.util.HashMap; + public class RavenDataRow { private int iRow; @@ -11,15 +14,16 @@ public class RavenDataRow { private double endT; private double f1; private double f2; - private int[] dataIndexes; +// private int[] dataIndexes; private boolean unpackOK; + private HashMap extraData = new HashMap<>(); // data on a row of raven data from a table. - public RavenDataRow(int iRow, String[] data, int[] dataIndexes) { + public RavenDataRow(RavenFileReader ravenReader, int iRow, String[] data) { this.iRow = iRow; this.data = data; - this.dataIndexes = dataIndexes; - unpackOK = unpackRow(dataIndexes); +// this.dataIndexes = dataIndexes; + unpackOK = unpackRow(ravenReader); } /** @@ -74,10 +78,12 @@ public class RavenDataRow { /** * Unpack the row into more useful columns using the column indexes. - * @param mainIndexes + * @param ravenReader * @return */ - private boolean unpackRow(int[] mainIndexes) { + private boolean unpackRow(RavenFileReader ravenReader) { + int[] mainIndexes = ravenReader.getMainIndexes(); + ArrayList extraColumns = ravenReader.getExtraColumns(); try { selection = getInteger(mainIndexes[0]); view = getString(mainIndexes[1]); @@ -86,12 +92,36 @@ public class RavenDataRow { endT = getDouble(mainIndexes[4]); f1 = getDouble(mainIndexes[5]); f2 = getDouble(mainIndexes[6]); + // and add all the extra data into a HashMap + if (extraColumns == null) { + return true; + } + for (RavenColumnInfo col : extraColumns) { + String data = getString(col.ravenTableIndex); + if (data != null) { + col.maxStrLength = Math.max(col.maxStrLength, data.length()); + extraData.put(col.name, data); + } + } + } catch (Exception e) { return false; } return true; } + + /** + * Get data from the extras map. + * @param name column name. + * @return + */ + public String getExtraData(String name) { + if (extraData == null) { + return null; + } + return extraData.get(name); + } /** * @return the selection @@ -142,17 +172,11 @@ public class RavenDataRow { return f2; } - /** - * @return the dataIndexes - */ - protected int[] getDataIndexes() { - return dataIndexes; - } - + /** * @return the unpackOK */ - protected boolean isUnpackOK() { + public boolean isUnpackOK() { return unpackOK; } @@ -165,4 +189,8 @@ public class RavenDataRow { return this.channel == oth.channel && this.beginT == oth.beginT && this.endT == oth.endT && this.f1 == oth.f1 && this.f2 == oth.f2; } + public HashMap getExtraData() { + return extraData; + } + } diff --git a/src/ravendata/RavenDataUnit.java b/src/ravendata/RavenDataUnit.java index 49929ccf..9f09f769 100644 --- a/src/ravendata/RavenDataUnit.java +++ b/src/ravendata/RavenDataUnit.java @@ -1,11 +1,17 @@ package ravendata; +import java.util.HashMap; +import java.util.Map.Entry; +import java.util.Set; + import PamDetection.PamDetection; import PamguardMVC.AcousticDataUnit; import PamguardMVC.DataUnitBaseData; import PamguardMVC.PamDataUnit; public class RavenDataUnit extends PamDataUnit implements AcousticDataUnit, PamDetection { + + private HashMap extraData; public RavenDataUnit(long timeMilliseconds, int channelMap, long durationMillis, double f1, double f2) { super(timeMilliseconds); @@ -15,6 +21,28 @@ public class RavenDataUnit extends PamDataUnit implements AcousticDataUnit, PamD setFrequency(freq); } + public HashMap getExtraData() { + return extraData; + } + + public void setExtraData(HashMap extraData) { + this.extraData = extraData; + } + + @Override + public String getSummaryString() { + String base = super.getSummaryString(); + if (extraData == null) { + return base; + } + Set> entries = extraData.entrySet(); + for (Entry e : entries) { + base += String.format("
%s: %s", e.getKey(), e.getValue()); + } + + return base; + } + } diff --git a/src/ravendata/RavenFileReader.java b/src/ravendata/RavenFileReader.java index f18e8eb0..bd93ed4c 100644 --- a/src/ravendata/RavenFileReader.java +++ b/src/ravendata/RavenFileReader.java @@ -5,6 +5,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; +import java.sql.Types; import java.util.ArrayList; public class RavenFileReader { @@ -13,6 +14,7 @@ public class RavenFileReader { private File ravenFile; private ArrayList columnNames; private BufferedReader fileReader; + private ArrayList extraColumns; private static String[] standardColumns = {"Selection", "View", "Channel", "Begin Time (s)", "End Time (s)", "Low Freq (Hz)", "High Freq (Hz)"}; // private static String[] otherColumns = {"Selection", "View", "Channel", "Begin Time (s)", "End Time (s)", "end", "Low Freq (Hz)", "High Freq (Hz)", @@ -37,6 +39,14 @@ private int columnErrors; } } + public ArrayList getExtraColumns() { + return extraColumns; + } + + public int[] getMainIndexes() { + return mainIndexes; + } + /** * open the file, read the file header and get a list of column names. * @throws IOException @@ -64,17 +74,107 @@ private int columnErrors; break; } String[] split = aLine.split("\t"); - RavenDataRow aRow = new RavenDataRow(iRow++, split, mainIndexes); + RavenDataRow aRow = new RavenDataRow(this, iRow++, split); allData.add(aRow); } + + checkExtraColumns(allData); + return allData; } + private void checkExtraColumns(ArrayList allData) { + if (extraColumns == null) { + return; + } + for (int i = 0; i < extraColumns.size(); i++) { + RavenColumnInfo col = extraColumns.get(i); + boolean allNull = isAllNull(col.name, allData); + if (allNull) { + continue; + } + boolean allInt = isAllInt(col.name, allData); + if (allInt) { + col.sqlType = Types.INTEGER; + continue; + } + boolean allDouble = isAllDouble(col.name, allData); + if (allDouble) { + col.sqlType = Types.DOUBLE; + continue; + } + // otherwise keep as string + col.sqlType = Types.CHAR; + } + + } + + /** + * Is data in every column null ? + * @param colName + * @param allData + * @return + */ + private boolean isAllNull(String colName, ArrayList allData) { + for (RavenDataRow aRow : allData) { + String xData = aRow.getExtraData(colName); + if (xData != null && xData.length() > 0) { + return false; + } + } + return true; + } + + /** + * Is data in every column either integer or null ? + * @param colName + * @param allData + * @return + */ + private boolean isAllInt(String colName, ArrayList allData) { + for (RavenDataRow aRow : allData) { + String xData = aRow.getExtraData(colName); + if (xData == null || xData.length() == 0) { + continue; + } + try { + int val = Integer.valueOf(xData); + } + catch (NumberFormatException e) { + return false; + } + } + return true; + } + + /** + * Is data in every column either integer or null ? + * @param colName + * @param allData + * @return + */ + private boolean isAllDouble(String colName, ArrayList allData) { + for (RavenDataRow aRow : allData) { + String xData = aRow.getExtraData(colName); + if (xData == null || xData.length() == 0) { + continue; + } + try { + double val = Double.valueOf(xData); + } + catch (NumberFormatException e) { + return false; + } + } + return true; + } + /** * Check we have the correct standard columns. */ private int checkColumnNames() { mainIndexes = new int[standardColumns.length]; + boolean[] isStandard = new boolean[columnNames.size()]; columnErrors = 0; for (int i = 0; i < standardColumns.length; i++) { mainIndexes[i] = columnNames.indexOf(standardColumns[i]); @@ -82,7 +182,18 @@ private int columnErrors; System.out.printf("Raven error: Unable to find column \"%s\" in data table\n", standardColumns[i]); columnErrors++; } + else { + isStandard[mainIndexes[i]] = true; + } } + // now find any extra columns hidden away in there. + extraColumns = new ArrayList<>(); + for (int i = 0; i < columnNames.size(); i++) { + if (isStandard[i] == false) { + extraColumns.add(new RavenColumnInfo(i, columnNames.get(i))); + } + } + return columnErrors; } diff --git a/src/ravendata/RavenLogging.java b/src/ravendata/RavenLogging.java index fab6c7de..8e81e39a 100644 --- a/src/ravendata/RavenLogging.java +++ b/src/ravendata/RavenLogging.java @@ -1,19 +1,23 @@ package ravendata; +import java.sql.Types; +import java.util.ArrayList; +import java.util.HashMap; + import PamDetection.AcousticSQLLogging; -import PamguardMVC.PamDataBlock; import PamguardMVC.PamDataUnit; +import generalDatabase.DBControlUnit; +import generalDatabase.PamTableDefinition; import generalDatabase.PamTableItem; -import generalDatabase.SQLLogging; import generalDatabase.SQLTypes; public class RavenLogging extends AcousticSQLLogging { private RavenDataBlock ravenDataBlock; private RavenControl ravenControl; + private ArrayList usedExtraColumns; + private PamTableItem[] extraItems; - private PamTableItem f1; - public RavenLogging(RavenControl ravenControl, RavenDataBlock pamDataBlock) { super(pamDataBlock, ravenControl.getUnitName()); this.ravenControl = ravenControl; @@ -21,9 +25,60 @@ public class RavenLogging extends AcousticSQLLogging { } @Override - protected PamDataUnit createDataUnit(SQLTypes sqlTypes, long timeMilliseconds, int chanMap, long duration, + protected RavenDataUnit createDataUnit(SQLTypes sqlTypes, long timeMilliseconds, int chanMap, long duration, double[] f) { - return new RavenDataUnit(timeMilliseconds, chanMap, duration, f[0], f[1]); + RavenDataUnit ravenDataUnit = new RavenDataUnit(timeMilliseconds, chanMap, duration, f[0], f[1]); + + if (extraItems == null || usedExtraColumns == null) { + return ravenDataUnit; + } + HashMap extraData = new HashMap<>(); + for (int i = 0; i < usedExtraColumns.size(); i++) { + String data = extraItems[i].getDeblankedStringValue(); + if (data != null && data.length() > 0) { + extraData.put(usedExtraColumns.get(i).name, data); + } + } + ravenDataUnit.setExtraData(extraData); + + return ravenDataUnit; + } + + public void addExtraColumns(ArrayList extraColumns) { + this.usedExtraColumns = new ArrayList(); + PamTableDefinition baseTable = getBaseTableDefinition(); + for (int i = 0; i < extraColumns.size(); i++) { + if (extraColumns.get(i).sqlType == null) { + continue; + } + usedExtraColumns.add(extraColumns.get(i)); + } + extraItems = new PamTableItem[usedExtraColumns.size()]; + for (int i = 0; i < usedExtraColumns.size(); i++) { + RavenColumnInfo col = usedExtraColumns.get(i); + extraItems[i] = new PamTableItem(col.name, Types.CHAR, Math.max(col.maxStrLength,30)); + baseTable.addTableItem(extraItems[i]); + } + setTableDefinition(baseTable); + DBControlUnit dbc = DBControlUnit.findDatabaseControl(); + if (dbc != null) { + dbc.getDbProcess().checkTable(baseTable); + } + } + + @Override + public void setTableData(SQLTypes sqlTypes, PamDataUnit pamDataUnit) { + super.setTableData(sqlTypes, pamDataUnit); + // and do the extras + RavenDataUnit ravenDataUnit = (RavenDataUnit) pamDataUnit; + HashMap extraData = ravenDataUnit.getExtraData(); + if (extraData == null || extraItems == null || usedExtraColumns == null) { + return; + } + for (int i = 0; i < usedExtraColumns.size(); i++) { + String data = extraData.get(usedExtraColumns.get(i).name); + extraItems[i].setValue(data); + } } diff --git a/src/ravendata/RavenParameters.java b/src/ravendata/RavenParameters.java index d5d4718f..93792833 100644 --- a/src/ravendata/RavenParameters.java +++ b/src/ravendata/RavenParameters.java @@ -1,10 +1,26 @@ package ravendata; import java.io.Serializable; +import java.util.ArrayList; public class RavenParameters implements Serializable, Cloneable { public static final long serialVersionUID = 1L; public String importFile; + + private ArrayList extraColumns; + + public ArrayList getExtraColumns() { + if (extraColumns == null) { + extraColumns = new ArrayList<>(); + } + return extraColumns; + } + + public void setExtraColumns(ArrayList extraColumns) { + this.extraColumns = extraColumns; + } + + } diff --git a/src/ravendata/RavenProcess.java b/src/ravendata/RavenProcess.java index 87187dbb..125eef1a 100644 --- a/src/ravendata/RavenProcess.java +++ b/src/ravendata/RavenProcess.java @@ -24,7 +24,7 @@ public class RavenProcess extends PamProcess { private RavenDataBlock ravenDataBlock; private RavenLogging ravenLogging; - + private static SymbolData standardSymbol = new SymbolData(PamSymbolType.SYMBOL_SQUARE, 20, 20, false, Color.white, Color.red); public RavenProcess(RavenControl pamControlledUnit) { @@ -43,14 +43,17 @@ public class RavenProcess extends PamProcess { @Override public void pamStart() { // TODO Auto-generated method stub - } @Override public void pamStop() { - // TODO Auto-generated method stub - + // TODO Auto-generated method stub } + + public RavenLogging getRavenLogging() { + return ravenLogging; + } + protected void createPAMGuardData(RavenFileReader fileReader, ArrayList ravenData) { /** * Need to find the acquisition module and then get detailed times of every file, not just @@ -84,6 +87,12 @@ public class RavenProcess extends PamProcess { ravenDataBlock.clearAll(); ravenLogging.deleteData(0, System.currentTimeMillis()*2); + /** + * Had to add an offset for some messed up Raven data. May or may not have to include + * this as an option in future releases. + */ + long offsetMillis = 0;//2843100; + RavenDataRow prevRow = null; for (RavenDataRow ravenRow : ravenData) { if (ravenRow.equals(prevRow) == false) { @@ -91,17 +100,19 @@ public class RavenProcess extends PamProcess { * A lot of Raven data appear twice, with different view values. * No need to import both. so only doing this if they are different. */ - int fileInd = getTimeIndex(ravenRow.getBeginT()*1000, absTime); + double ravenStart = ravenRow.getBeginT()*1000 + offsetMillis; + int fileInd = getTimeIndex(ravenStart, absTime); if (fileInd == absTime.length) { String msg = String.format("Data at time %6.4f is beyond the end of available sound file data", ravenRow.getBeginT()); WarnOnce.showWarning("Error importing RAVEN data", msg, WarnOnce.WARNING_MESSAGE); break; } long fileStart = fileStarts[fileInd]; - long absStart = fileStart + (long) (ravenRow.getBeginT()*1000.)-absTime[fileInd]; + long absStart = fileStart + (long) (ravenStart)-absTime[fileInd]; long duration = (long) ((ravenRow.getEndT()-ravenRow.getBeginT())*1000.); int chanMap = 1<<(ravenRow.getChannel()-1); RavenDataUnit rdu = new RavenDataUnit(absStart, chanMap, duration, ravenRow.getF1(), ravenRow.getF2()); + rdu.setExtraData(ravenRow.getExtraData()); getRavenDataBlock().addPamData(rdu); ravenLogging.logData(DBControlUnit.findConnection(), rdu); }