From 63cc66a4b197ce86087f57edc38095161896b79f Mon Sep 17 00:00:00 2001 From: Douglas Gillespie <50671166+douggillespie@users.noreply.github.com> Date: Tue, 18 Oct 2022 17:04:16 +0100 Subject: [PATCH] Merge from DG (#76) * Work on bug fixes in CTD classifier * couple of final neatenings up of CTD * DWV Import Fixed DWV importer so that a binary file is still created if a DWV file doesn't exist. this is totally normal in quiet conditions, so the correct thing to do is to generate a binary file with no data so that effort is correctly recorded. --- src/soundtrap/DWVConvertInformation.java | 32 +++++++++-- src/soundtrap/DWVConverter.java | 72 ++++++++++++++++++------ src/soundtrap/ImportBCLDialog.java | 20 ++++--- 3 files changed, 93 insertions(+), 31 deletions(-) diff --git a/src/soundtrap/DWVConvertInformation.java b/src/soundtrap/DWVConvertInformation.java index f18caea7..bb0da67b 100644 --- a/src/soundtrap/DWVConvertInformation.java +++ b/src/soundtrap/DWVConvertInformation.java @@ -9,13 +9,17 @@ public class DWVConvertInformation { private STGroupInfo fileGroup; private int nDWV; - private int nDone; + private int nDWVDone; + private int nFileDone; + private int nFile; - public DWVConvertInformation(STGroupInfo fileGroup, int nDWV, int nDone) { + public DWVConvertInformation(STGroupInfo fileGroup, int nFile, int nFileDone, int nDWV, int nDWVDone) { this.fileGroup = fileGroup; + this.nFile = nFile; + this.nFileDone = nFileDone; this.nDWV = nDWV; - this.nDone = nDone; + this.nDWVDone = nDWVDone; } @@ -36,10 +40,26 @@ public class DWVConvertInformation { /** - * @return the nDone + * @return the nDWVDone */ - public int getnDone() { - return nDone; + public int getnDWVDone() { + return nDWVDone; + } + + + /** + * @return the nFileDone + */ + public int getnFileDone() { + return nFileDone; + } + + + /** + * @return the nFile + */ + public int getnFile() { + return nFile; } } diff --git a/src/soundtrap/DWVConverter.java b/src/soundtrap/DWVConverter.java index 1c9d1bc4..ca4ba9d0 100644 --- a/src/soundtrap/DWVConverter.java +++ b/src/soundtrap/DWVConverter.java @@ -81,8 +81,10 @@ public class DWVConverter { binarySource.setBinaryStorageStream(outputStream); binaryStream = clickBinaryDataSource.getBinaryStorageStream(); + int iFile = 0; + int nFile = fileGroups.size(); for (STGroupInfo fileGroup:fileGroups) { - processFiles(fileGroup); + processFiles(fileGroup, nFile, ++iFile); if (keepRunning == false) { break; } @@ -94,16 +96,30 @@ public class DWVConverter { return null; } - private void processFiles(STGroupInfo fileGroup) { - this.publish(new DWVConvertInformation(fileGroup, 0, 0)); - if (fileGroup.hasDWV() == false) return; + private void processFiles(STGroupInfo fileGroup, int nFile, int iFile) { + this.publish(new DWVConvertInformation(fileGroup, nFile, iFile, 0, 0)); + if (fileGroup.hasDWV() == false) { + /* + * Don't do this. In quiet conditions there will be a BCL file with a start and stop time + * but if there were no detections there will be no dwv file. We should make the pgdf file + * in any case so that there is a full PAMGuard record of effort. + * In other circumstances dwv files don't actually have data, in which case there may still be + * a null file, which is not so good. Will have to think about how to report this - perhaps in + * this case it's better not to make a pgdf file to show a genuine gap in the data. + */ +// return; + } BCLReader bclReader = new BCLReader(fileGroup.getBclFile()); boolean ok = bclReader.open(); if (ok == false) return; - DWVReader dwvReader = new DWVReader(fileGroup.getDwvFile(), fileGroup.getDwvInfo()); - dwvReader.openDWV(); - int nDWV = dwvReader.getNumDWV(); - int repStep = Math.max(1, nDWV/100); + int nDWV = 0; + DWVReader dwvReader = null; + if (fileGroup.hasDWV()) { + dwvReader = new DWVReader(fileGroup.getDwvFile(), fileGroup.getDwvInfo()); + dwvReader.openDWV(); + nDWV = dwvReader.getNumDWV(); + } + int repStep = Math.max(10, nDWV/100); int nRead = 0; while (true) { BCLLine bclLine = bclReader.nextBCLLine(); @@ -127,7 +143,7 @@ public class DWVConverter { dataLine(fileGroup, bclLine, dwvReader); nRead++; if (nRead%repStep == 0) { - this.publish(new DWVConvertInformation(fileGroup, nDWV, nRead)); + this.publish(new DWVConvertInformation(fileGroup, nFile, iFile, nDWV, nRead)); } break; @@ -140,10 +156,14 @@ public class DWVConverter { // e.printStackTrace(); // } } + this.publish(new DWVConvertInformation(fileGroup, nFile, iFile, nDWV, nDWV)); } private void dataLine(STGroupInfo fileGroup, BCLLine bclLine, DWVReader dwvReader) { - double[] dwvData = dwvReader.readNextClick(null); + double[] dwvData = null; + if (dwvReader != null) { + dwvData = dwvReader.readNextClick(null); + } if (dwvData == null) { return; // can happen if file didn't flush correclty and some dwv is missing. } @@ -185,21 +205,37 @@ public class DWVConverter { } if (bclLine.getState() == 1) { - long dwvFileStart = fileGroup.getDwvInfo().getTimeInfo().samplingStartTimeUTC; - long wavFileStart = fileGroup.getWavInfo().getTimeInfo().samplingStartTimeUTC; fileStartMicroseconds = bclLine.getMicroSeconds(); - Debug.out.printf("DWV start %s wav start %s bcl start %s\n", PamCalendar.formatDBDateTime(dwvFileStart), - PamCalendar.formatDBDateTime(wavFileStart), PamCalendar.formatDBDateTime(bclLine.getMilliseconds())); + long wavFileStart = fileGroup.getWavInfo().getTimeInfo().samplingStartTimeUTC; + try { + if (fileGroup.getDwvInfo() != null) { + long dwvFileStart = fileGroup.getDwvInfo().getTimeInfo().samplingStartTimeUTC; + Debug.out.printf("DWV start %s wav start %s bcl start %s\n", PamCalendar.formatDBDateTime(dwvFileStart), + PamCalendar.formatDBDateTime(wavFileStart), PamCalendar.formatDBDateTime(bclLine.getMilliseconds())); + } + else { + Debug.out.printf("No DWV File !!! wav start %s bcl start %s\n", + PamCalendar.formatDBDateTime(wavFileStart), PamCalendar.formatDBDateTime(bclLine.getMilliseconds())); + } + } + catch (Exception e) { + System.out.println(e.getMessage()); + } binaryStream.openOutputFiles(bclLine.getMilliseconds()); binaryStream.writeHeader(bclLine.getMilliseconds(), System.currentTimeMillis()); binaryStream.writeModuleHeader(); onEffort = true; } if (bclLine.getState() == 0) { - long dwvFileStop = fileGroup.getDwvInfo().getTimeInfo().samplingStopTimeUTC; - long wavFileStop = fileGroup.getWavInfo().getTimeInfo().samplingStopTimeUTC; - Debug.out.printf("DWV stop %s wav stop %s E stop %s\n", PamCalendar.formatDBDateTime(dwvFileStop), - PamCalendar.formatDBDateTime(wavFileStop), PamCalendar.formatDBDateTime(bclLine.getMilliseconds())); + try { + long dwvFileStop = fileGroup.getDwvInfo().getTimeInfo().samplingStopTimeUTC; + long wavFileStop = fileGroup.getWavInfo().getTimeInfo().samplingStopTimeUTC; + Debug.out.printf("DWV stop %s wav stop %s E stop %s\n", PamCalendar.formatDBDateTime(dwvFileStop), + PamCalendar.formatDBDateTime(wavFileStop), PamCalendar.formatDBDateTime(bclLine.getMilliseconds())); + } + catch (Exception e) { + System.out.println(e.getMessage()); + } binaryStream.writeModuleFooter(); binaryStream.writeFooter(bclLine.getMilliseconds(), System.currentTimeMillis(), BinaryFooter.END_UNKNOWN); binaryStream.closeFile(); diff --git a/src/soundtrap/ImportBCLDialog.java b/src/soundtrap/ImportBCLDialog.java index bff30c88..3582c646 100644 --- a/src/soundtrap/ImportBCLDialog.java +++ b/src/soundtrap/ImportBCLDialog.java @@ -56,7 +56,7 @@ public class ImportBCLDialog extends PamDialog { private JTextField customDateTimeFormat; private JLabel fileCountInfo; private JLabel thisFileInfo; - private JProgressBar allProgress; + private JProgressBar oneFileProgress, totalProgress; private JComboBox detectorName; private int nSoundTraps; private Hashtable uniqueDevices = new Hashtable<>(); @@ -162,8 +162,10 @@ public class ImportBCLDialog extends PamDialog { thisFileInfo = new JLabel(" - "); progressPanel.add(thisFileInfo); - allProgress = new JProgressBar(); - progressPanel.add(allProgress); + totalProgress = new JProgressBar(); + progressPanel.add(totalProgress); + oneFileProgress = new JProgressBar(); + progressPanel.add(oneFileProgress); startButton = new JButton("Start"); getButtonPanel().add(startButton, 0); @@ -388,10 +390,14 @@ public class ImportBCLDialog extends PamDialog { thisFileInfo.setText("NO DWV records"); return; } - int prog = dwvConvertInformation.getnDone() * 100 / dwvConvertInformation.getnDWV(); - allProgress.setValue(prog); - thisFileInfo.setText(String.format("Repacked %d of %d clicks", dwvConvertInformation.getnDone(), - dwvConvertInformation.getnDWV())); + int prog = dwvConvertInformation.getnDWVDone() * 100 / dwvConvertInformation.getnDWV(); + oneFileProgress.setValue(prog); + thisFileInfo.setText(String.format("Repacked %d of %d clicks in file %d of %d", + dwvConvertInformation.getnDWVDone(), dwvConvertInformation.getnDWV(), + dwvConvertInformation.getnFileDone(), dwvConvertInformation.getnFile())); + + prog = dwvConvertInformation.getnFileDone() * 100 / dwvConvertInformation.getnFile(); + totalProgress.setValue(prog); } @Override