From 22531824388699be005e53d7ddd1ecec50b4449b Mon Sep 17 00:00:00 2001 From: Douglas Gillespie <50671166+douggillespie@users.noreply.github.com> Date: Tue, 15 Nov 2022 17:34:10 +0000 Subject: [PATCH 1/5] Version 2.02.06 (hopefully) --- pom.xml | 2 +- src/Filters/IIRFilterMethod.java | 3 +++ src/PamController/PamguardVersionInfo.java | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 9fb8bdd6..4a00c145 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 org.pamguard Pamguard - 2.02.05a + 2.02.06 Pamguard Java12+ Pamguard for Java 12+, using Maven to control dependcies www.pamguard.org diff --git a/src/Filters/IIRFilterMethod.java b/src/Filters/IIRFilterMethod.java index 44dd4c2f..bd902e69 100644 --- a/src/Filters/IIRFilterMethod.java +++ b/src/Filters/IIRFilterMethod.java @@ -313,6 +313,9 @@ public abstract class IIRFilterMethod extends FilterMethod { // if (nOddOnes != 0) { pole1 = pzPairs[i].getPole(); zero1 = pzPairs[i].getZero(); + if (pole1 == null || zero1 == null) { + continue; + } a1 = -zero1.real; b1 = pole1.real; doubleCoefficients[j+0] = a1; diff --git a/src/PamController/PamguardVersionInfo.java b/src/PamController/PamguardVersionInfo.java index 270b31df..68b68f42 100644 --- a/src/PamController/PamguardVersionInfo.java +++ b/src/PamController/PamguardVersionInfo.java @@ -31,12 +31,12 @@ public class PamguardVersionInfo { * Version number, major version.minorversion.sub-release. * Note: can't go higher than sub-release 'f' */ - static public final String version = "2.02.05a"; + static public final String version = "2.02.06"; /** * Release date */ - static public final String date = "11 November 2022"; + static public final String date = "15 November 2022"; // /** // * Release type - Beta or Core From 75b05a25e6aed570cddd340828ff849eb6f50a45 Mon Sep 17 00:00:00 2001 From: Douglas Gillespie <50671166+douggillespie@users.noreply.github.com> Date: Sun, 20 Nov 2022 21:00:00 +0000 Subject: [PATCH 2/5] ROCCA Memory leak Another attempt at healing ROCCA memory leak --- src/PamguardMVC/PamDataBlock.java | 23 +++++++++++++++++++++-- src/PamguardMVC/PamObservable.java | 6 +++--- src/rocca/RoccaContourDataBlock.java | 24 +++++++++++++++++++++++- src/rocca/RoccaProcess.java | 2 +- 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/src/PamguardMVC/PamDataBlock.java b/src/PamguardMVC/PamDataBlock.java index e0c3b9f7..68a4ab25 100644 --- a/src/PamguardMVC/PamDataBlock.java +++ b/src/PamguardMVC/PamDataBlock.java @@ -391,7 +391,7 @@ public class PamDataBlock extends PamObservable { isNetworkReceive = PamController.getInstance().getRunMode() == PamController.RUN_NETWORKRECEIVER; if (!isOffline) { - t.start(); + removeTimer.start(); } } @@ -469,7 +469,7 @@ public class PamDataBlock extends PamObservable { // } } - Timer t = new Timer(500, new ActionListener() { + Timer removeTimer = new Timer(500, new ActionListener() { public void actionPerformed(ActionEvent evt) { int n; if (shouldDelete()) { @@ -2413,6 +2413,25 @@ public class PamDataBlock extends PamObservable { return unitClass; } + /** + * clean up datablock when it's no longer needed + */ + public void dispose() { + stopTimer(); + clearAll(); + } + + /** + * Had some issues with the Timer holding a reference to the underlying PamDataBlock + * (RoccaContourDataBlock, in this case) and not releasing it for garbage collection. + * Added in this method to force the timer to stop and release it's hold. + */ + @Override + public void stopTimer() { + super.stopTimer(); + removeTimer.stop(); + } + public void autoSetDataBlockMixMode() { if (AcousticDataUnit.class.isAssignableFrom(unitClass)) { // System.out.println(unitClass + " is acoustic data" ); diff --git a/src/PamguardMVC/PamObservable.java b/src/PamguardMVC/PamObservable.java index da6a540b..0d693c99 100644 --- a/src/PamguardMVC/PamObservable.java +++ b/src/PamguardMVC/PamObservable.java @@ -93,7 +93,7 @@ public class PamObservable {//extends PanelOverlayDraw { pamObservers = new ArrayList(); instantObservers = new ArrayList(); pamProfiler = PamProfiler.getInstance(); - t.start(); + cpuTimer.start(); } /** @@ -400,7 +400,7 @@ public class PamObservable {//extends PanelOverlayDraw { } } - private Timer t = new Timer(4321, new ActionListener() { + private Timer cpuTimer = new Timer(4321, new ActionListener() { public void actionPerformed(ActionEvent evt) { long now = System.currentTimeMillis(); if (cpuUsage == null) return; @@ -420,7 +420,7 @@ public class PamObservable {//extends PanelOverlayDraw { * Added in this method to force the timer to stop and release it's hold. */ public void stopTimer() { - t.stop(); + cpuTimer.stop(); } public double getCPUPercent(int objectIndex) { diff --git a/src/rocca/RoccaContourDataBlock.java b/src/rocca/RoccaContourDataBlock.java index b295088d..1672c93d 100644 --- a/src/rocca/RoccaContourDataBlock.java +++ b/src/rocca/RoccaContourDataBlock.java @@ -1574,6 +1574,28 @@ public class RoccaContourDataBlock extends PamDataBlock { public double[][] getWavInfo() { return wavInfo; } - + + +// int nS, nT; +// @Override +// protected int removeOldUnitsT(long currentTimeMS) { +// // TODO Auto-generated method stub +// nT++; +// System.out.printf("Enter removeoldUnitsT, entry count %d\n", nT); +// int n = super.removeOldUnitsT(currentTimeMS); +// nT--; +// return n; +// } +// +// @Override +// protected int removeOldUnitsS(long mastrClockSample) { +// // TODO Auto-generated method stub +// nS++; +// System.out.printf("Enter removeoldUnitsS, entry count %d\n", nS); +// int n = super.removeOldUnitsS(mastrClockSample); +// nS--; +// return n; +// } +// } diff --git a/src/rocca/RoccaProcess.java b/src/rocca/RoccaProcess.java index 71847953..7abe2577 100644 --- a/src/rocca/RoccaProcess.java +++ b/src/rocca/RoccaProcess.java @@ -544,7 +544,7 @@ public class RoccaProcess extends PamProcess { updateSidePanel(rcdb, true); saveContourStats(rcdb, rcdb.getChannelMap(), numDetections, sNum); // rcdb.setNaturalLifetimeMillis(0); - rcdb.stopTimer(); + rcdb.dispose(); rcdb = null; } From f910d805179e1b7b0810d88fbb74780f78d100f8 Mon Sep 17 00:00:00 2001 From: Douglas Gillespie <50671166+douggillespie@users.noreply.github.com> Date: Mon, 20 Feb 2023 17:15:54 +0000 Subject: [PATCH 3/5] Reprocessing options Analysis of input and output files and can restart from where it left off when processing an offline dataset. --- src/Acquisition/AcquisitionControl.java | 26 +- src/Acquisition/AcquisitionProcess.java | 39 +- src/Acquisition/FileInputSystem.java | 1 + src/Acquisition/FolderInputSystem.java | 52 ++- .../offlineFuncs/OfflineWavFileServer.java | 2 +- src/PamController/DataInputStore.java | 31 ++ src/PamController/DataOutputStore.java | 32 ++ src/PamController/InputStoreInfo.java | 81 ++++ src/PamController/OfflineDataStore.java | 4 +- src/PamController/PamController.java | 42 +- src/PamController/RawInputControlledUnit.java | 21 + .../fileprocessing/ReprocessChoiceDialog.java | 116 ++++++ .../fileprocessing/ReprocessManager.java | 273 +++++++++++++ .../fileprocessing/ReprocessStoreChoice.java | 53 +++ .../fileprocessing/StoreChoiceSummary.java | 234 +++++++++++ .../fileprocessing/StoreStatus.java | 115 ++++++ src/PamUtils/worker/filelist/WavFileType.java | 12 +- src/PamguardMVC/uid/binaryUIDFunctions.java | 3 +- src/binaryFileStorage/BinaryStore.java | 46 ++- src/binaryFileStorage/BinaryStoreDeleter.java | 363 ++++++++++++++++++ src/binaryFileStorage/BinaryStoreStatus.java | 166 ++++++++ .../BinaryStoreStatusFuncs.java | 270 +++++++++++++ src/cpod/CPODControl.java | 3 +- src/d3/D3Control.java | 2 + src/decimator/DecimatorControl.java | 3 +- src/difar/beamforming/BeamformControl.java | 2 + src/generalDatabase/DBControlUnit.java | 14 +- src/generalDatabase/DBProcess.java | 143 +++++++ src/generalDatabase/DatabaseStoreStatus.java | 43 +++ src/generalDatabase/SQLLogging.java | 15 +- src/pamguard/Pamguard.java | 10 + 31 files changed, 2194 insertions(+), 23 deletions(-) create mode 100644 src/PamController/DataInputStore.java create mode 100644 src/PamController/DataOutputStore.java create mode 100644 src/PamController/InputStoreInfo.java create mode 100644 src/PamController/RawInputControlledUnit.java create mode 100644 src/PamController/fileprocessing/ReprocessChoiceDialog.java create mode 100644 src/PamController/fileprocessing/ReprocessManager.java create mode 100644 src/PamController/fileprocessing/ReprocessStoreChoice.java create mode 100644 src/PamController/fileprocessing/StoreChoiceSummary.java create mode 100644 src/PamController/fileprocessing/StoreStatus.java create mode 100644 src/binaryFileStorage/BinaryStoreDeleter.java create mode 100644 src/binaryFileStorage/BinaryStoreStatus.java create mode 100644 src/binaryFileStorage/BinaryStoreStatusFuncs.java create mode 100644 src/generalDatabase/DatabaseStoreStatus.java diff --git a/src/Acquisition/AcquisitionControl.java b/src/Acquisition/AcquisitionControl.java index 150fc411..f0bd367d 100644 --- a/src/Acquisition/AcquisitionControl.java +++ b/src/Acquisition/AcquisitionControl.java @@ -62,6 +62,8 @@ import Acquisition.sud.SUDNotificationManager; import Array.ArrayManager; import Array.PamArray; import Array.Preamplifier; +import PamController.DataInputStore; +import PamController.InputStoreInfo; import PamController.OfflineFileDataStore; import PamController.PamControlledUnit; import PamController.PamControlledUnitGUI; @@ -71,6 +73,8 @@ import PamController.PamControllerInterface; import PamController.PamGUIManager; import PamController.PamSettingManager; import PamController.PamSettings; +import PamController.RawInputControlledUnit; +import PamController.fileprocessing.StoreStatus; import PamModel.PamModel; import PamModel.SMRUEnable; import PamUtils.FrequencyFormat; @@ -100,7 +104,7 @@ import PamguardMVC.dataOffline.OfflineDataLoadInfo; * @see Acquisition.DaqSystem * */ -public class AcquisitionControl extends PamControlledUnit implements PamSettings, OfflineFileDataStore { +public class AcquisitionControl extends RawInputControlledUnit implements PamSettings, OfflineFileDataStore, DataInputStore { protected ArrayList systemList; @@ -849,4 +853,24 @@ public class AcquisitionControl extends PamControlledUnit implements PamSettings return sudNotificationManager; } + + @Override + public int getRawInputType() { + DaqSystem system = acquisitionProcess.getRunningSystem(); + if (system == null) { + return RAW_INPUT_UNKNOWN; + } + else { + return system.isRealTime() ? RAW_INPUT_REALTIME : RAW_INPUT_FILEARCHIVE; + } + } + @Override + public InputStoreInfo getStoreInfo(boolean detail) { + return getDaqProcess().getStoreInfo(detail); + } + @Override + public boolean setAnalysisStartTime(long startTime) { + return getDaqProcess().setAnalysisStartTime(startTime); + } + } diff --git a/src/Acquisition/AcquisitionProcess.java b/src/Acquisition/AcquisitionProcess.java index ef896874..c5e331fc 100644 --- a/src/Acquisition/AcquisitionProcess.java +++ b/src/Acquisition/AcquisitionProcess.java @@ -1,7 +1,9 @@ package Acquisition; +import java.awt.Window; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; +import java.util.ArrayList; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -19,7 +21,13 @@ import Filters.FilterBand; import Filters.FilterParams; import Filters.FilterType; import Filters.IirfFilter; +import PamController.DataInputStore; +import PamController.InputStoreInfo; +import PamController.OfflineDataStore; +import PamController.PamControlledUnit; import PamController.PamController; +import PamController.fileprocessing.ReprocessManager; +import PamController.fileprocessing.StoreStatus; import PamController.status.BaseProcessCheck; import PamController.status.ProcessCheck; import PamDetection.RawDataUnit; @@ -36,7 +44,10 @@ import PamguardMVC.PamProcess; import PamguardMVC.PamRawDataBlock; import PamguardMVC.RequestCancellationObject; import PamguardMVC.dataOffline.OfflineDataLoadInfo; +import dataGram.DatagramManager; +import dataMap.OfflineDataMapPoint; import pamScrollSystem.AbstractScrollManager; +import pamScrollSystem.ViewLoadObserver; /** * Data acquisition process for all types of input device. @@ -54,7 +65,7 @@ import pamScrollSystem.AbstractScrollManager; * @see PamguardMVC.PamDataUnit * */ -public class AcquisitionProcess extends PamProcess { +public class AcquisitionProcess extends PamProcess implements DataInputStore { public static final int LASTDATA = 2; // don't use zero since need to see if no notification has been received. @@ -523,12 +534,12 @@ public class AcquisitionProcess extends PamProcess { System.out.printf("Unable to find daq system %s\n", acquisitionControl.acquisitionParameters.daqSystemType); return; } - - + systemPrepared = runningSystem.prepareSystem(acquisitionControl); } + @Override public void setSampleRate(float sampleRate, boolean notify) { acquisitionControl.acquisitionParameters.sampleRate = sampleRate; @@ -1223,6 +1234,28 @@ public class AcquisitionProcess extends PamProcess { public PamDataBlock getDaqStatusDataBlock() { return daqStatusDataBlock; } + + @Override + public InputStoreInfo getStoreInfo(boolean detail) { + if (runningSystem instanceof DataInputStore) { + return ((DataInputStore) runningSystem).getStoreInfo(detail); + } + else { + return null; + } + } + + @Override + public boolean setAnalysisStartTime(long startTime) { + if (runningSystem instanceof DataInputStore) { + return ((DataInputStore) runningSystem).setAnalysisStartTime(startTime); + } + else { + return false; + } + } + + } diff --git a/src/Acquisition/FileInputSystem.java b/src/Acquisition/FileInputSystem.java index 1b15482f..34f49ffa 100644 --- a/src/Acquisition/FileInputSystem.java +++ b/src/Acquisition/FileInputSystem.java @@ -1146,6 +1146,7 @@ public class FileInputSystem extends DaqSystem implements ActionListener, PamSe protected void fileListComplete() { if (GlobalArguments.getParam(PamController.AUTOEXIT) != null) { System.out.println("All sound files processed, PAMGuard can close on " + PamController.AUTOEXIT); + PamController.getInstance().setPamStatus(PamController.PAM_COMPLETE); PamController.getInstance().batchProcessingComplete(); } } diff --git a/src/Acquisition/FolderInputSystem.java b/src/Acquisition/FolderInputSystem.java index 92ee94d7..9ac7778c 100644 --- a/src/Acquisition/FolderInputSystem.java +++ b/src/Acquisition/FolderInputSystem.java @@ -33,6 +33,8 @@ import pamguard.GlobalArguments; import Acquisition.pamAudio.PamAudioFileManager; import Acquisition.pamAudio.PamAudioFileFilter; import Acquisition.pamAudio.PamAudioSystem; +import PamController.DataInputStore; +import PamController.InputStoreInfo; import PamController.PamControlledUnitSettings; import PamController.PamController; import PamController.PamSettings; @@ -57,7 +59,7 @@ import PamView.panel.PamProgressBar; * @author Doug Gillespie * */ -public class FolderInputSystem extends FileInputSystem implements PamSettings{ +public class FolderInputSystem extends FileInputSystem implements PamSettings, DataInputStore { // Timer timer; public static final String daqType = "File Folder Acquisition System"; @@ -138,6 +140,8 @@ public class FolderInputSystem extends FileInputSystem implements PamSettings{ } String[] selList = {globalFolder}; folderInputParameters.setSelectedFiles(selList); + // need to immediately make the allfiles list since it's about to get used by the reprocess manager + makeSelFileList(); } /** @@ -827,5 +831,51 @@ public class FolderInputSystem extends FileInputSystem implements PamSettings{ folderInputPane.setParams(folderInputParameters); } + @Override + public InputStoreInfo getStoreInfo(boolean detail) { + if (allFiles == null || allFiles.size() == 0) { + return null; + } + WavFileType firstFile = allFiles.get(0); + long firstFileStart = getFileStartTime(firstFile.getAbsoluteFile()); + WavFileType lastFile = allFiles.get(allFiles.size()-1); + long lastFileStart = getFileStartTime(lastFile.getAbsoluteFile()); + lastFile.getAudioInfo(); + long lastFileEnd = (long) (lastFileStart + lastFile.getDurationInSeconds()*1000.); + InputStoreInfo storeInfo = new InputStoreInfo(acquisitionControl, allFiles.size(), firstFileStart, lastFileStart, lastFileEnd); + if (detail) { + long[] allFileStarts = new long[allFiles.size()]; + for (int i = 0; i < allFiles.size(); i++) { + allFileStarts[i] = getFileStartTime(allFiles.get(i).getAbsoluteFile()); + } + storeInfo.setFileStartTimes(allFileStarts); + } + return storeInfo; + } + + @Override + public boolean setAnalysisStartTime(long startTime) { + /** + * Called from the reprocess manager just before PAMGuard starts with a time + * we want to process from. This should be equal to the start of one of the files + * so all we have to do (in principle) is to set the currentfile to that index and + * processing will continue from there. + */ + if (allFiles == null || allFiles.size() == 0) { + return false; + } + for (int i = 0; i < allFiles.size(); i++) { + long fileStart = getFileStartTime(allFiles.get(i).getAbsoluteFile()); + if (fileStart >= startTime) { + currentFile = i; + System.out.printf("Sound Acquisition start processing at file %s time %s\n", allFiles.get(i).getName(), + PamCalendar.formatDBDateTime(fileStart)); + return true; + } + } + + return false; + } + } diff --git a/src/Acquisition/offlineFuncs/OfflineWavFileServer.java b/src/Acquisition/offlineFuncs/OfflineWavFileServer.java index c1b6efa6..b10b27dc 100644 --- a/src/Acquisition/offlineFuncs/OfflineWavFileServer.java +++ b/src/Acquisition/offlineFuncs/OfflineWavFileServer.java @@ -17,6 +17,7 @@ import Acquisition.pamAudio.PamAudioFileManager; import Acquisition.pamAudio.PamAudioFileLoader; import Acquisition.pamAudio.PamAudioFileFilter; import PamController.OfflineFileDataStore; +import PamController.fileprocessing.StoreStatus; import PamguardMVC.PamDataBlock; import PamguardMVC.dataOffline.OfflineDataLoadInfo; import dataMap.OfflineDataMap; @@ -188,5 +189,4 @@ public class OfflineWavFileServer extends OfflineFileServer { } - } diff --git a/src/PamController/DataInputStore.java b/src/PamController/DataInputStore.java new file mode 100644 index 00000000..af4f5a4b --- /dev/null +++ b/src/PamController/DataInputStore.java @@ -0,0 +1,31 @@ +package PamController; + +/** + * Functions for a data input store. There is a fair bit of overlap for this and + * OfflineDataStore, but the OfflineDataStore is really about stuff that can provide + * data offline which needs mapping. This is specifically about data which will be input + * during 'normal operation, i.e. sound acquisition and Tritech sonar data + * (a plugin, but coming down the tracks at us all). + * @author dg50 + * @see OfflineDataStore + * @See DataOutputStore + * + */ +public interface DataInputStore { + + /** + * Get information about the input store (e.g. start times of all files). + * @param detail + * @return information about data input. + */ + public InputStoreInfo getStoreInfo(boolean detail); + + /** + * Set an analysis start time. This might get called just before + * processing starts, in which case + * @param startTime + * @return ok if no problems. + */ + public boolean setAnalysisStartTime(long startTime); + +} diff --git a/src/PamController/DataOutputStore.java b/src/PamController/DataOutputStore.java new file mode 100644 index 00000000..a3536410 --- /dev/null +++ b/src/PamController/DataOutputStore.java @@ -0,0 +1,32 @@ +package PamController; + +import PamController.fileprocessing.StoreStatus; + +/** + * Functions for a data output store. there is a fair bit of overlap for this and + * OfflineDataStore, but the OfflineDataStore is really about stuff that can provide + * data offline which needs mapping. This is specifically about data which will be stored + * during 'normal operation, i.e. binary and database modules. + * @author dg50 + * @see OfflineDataStore + * @See DataInputStore + * + */ +public interface DataOutputStore extends OfflineDataStore { + + /** + * Get the store status, i.e. does it exist, does it contain data, if so over what date range, + * etc. + * @param getDetail + * @return + */ + public StoreStatus getStoreStatus(boolean getDetail); + + /** + * Delete all data from a given time, in all data streams. + * @param timeMillis time to delete from (anything >= this time) + * @return true if it seems to have worked OK. False if any errors (e.g. database or file system error). + */ + public boolean deleteDataFrom(long timeMillis); + +} diff --git a/src/PamController/InputStoreInfo.java b/src/PamController/InputStoreInfo.java new file mode 100644 index 00000000..265ed9d2 --- /dev/null +++ b/src/PamController/InputStoreInfo.java @@ -0,0 +1,81 @@ +package PamController; + +import PamUtils.PamCalendar; + +public class InputStoreInfo { + + private DataInputStore dataInputStore; + private int nFiles; + private long firstFileStart, lastFileStart, lastFileEnd; + private long[] fileStartTimes; + + public InputStoreInfo(DataInputStore dataInputStore, int nFiles, long firstFileStart, long lastFileStart, long lastFileEnd) { + super(); + this.dataInputStore = dataInputStore; + this.nFiles = nFiles; + this.firstFileStart = firstFileStart; + this.lastFileStart = lastFileStart; + this.lastFileEnd = lastFileEnd; + } + + /** + * @return the nFiles + */ + public int getnFiles() { + return nFiles; + } + + /** + * @return the firstFileStart + */ + public long getFirstFileStart() { + return firstFileStart; + } + + /** + * @return the lastFileStart + */ + public long getLastFileStart() { + return lastFileStart; + } + + /** + * @return the lastFileEnd + */ + public long getLastFileEnd() { + return lastFileEnd; + } + + @Override + public String toString() { + return String.format("%s: %d files. First start %s, last start %s, last end %s", dataInputStore.getClass().getName(), nFiles, + PamCalendar.formatDBDateTime(firstFileStart), PamCalendar.formatDBDateTime(lastFileStart), + PamCalendar.formatDBDateTime(lastFileEnd)); + } + + /** + * @return the dataInputStore + */ + public DataInputStore getDataInputStore() { + return dataInputStore; + } + + /** + * Set the start times of all files in data set. + * @param allFileStarts + */ + public void setFileStartTimes(long[] allFileStarts) { + this.fileStartTimes = allFileStarts; + + } + + /** + * @return the fileStartTimes + */ + public long[] getFileStartTimes() { + return fileStartTimes; + } + + + +} diff --git a/src/PamController/OfflineDataStore.java b/src/PamController/OfflineDataStore.java index 86e2b367..52e64150 100644 --- a/src/PamController/OfflineDataStore.java +++ b/src/PamController/OfflineDataStore.java @@ -2,6 +2,7 @@ package PamController; import java.awt.Window; +import PamController.fileprocessing.StoreStatus; import dataGram.DatagramManager; import dataMap.OfflineDataMapPoint; import pamScrollSystem.ViewLoadObserver; @@ -52,7 +53,7 @@ public interface OfflineDataStore { /** * Moved this function over from binary data store. - * Many storage systems may notbe able to do this, but some might ! + * Many storage systems may not be able to do this, but some might ! * @param dataBlock * @param dmp * @return @@ -63,4 +64,5 @@ public interface OfflineDataStore { * @return the datagramManager */ public DatagramManager getDatagramManager(); + } diff --git a/src/PamController/PamController.java b/src/PamController/PamController.java index c7546e04..009af1bd 100644 --- a/src/PamController/PamController.java +++ b/src/PamController/PamController.java @@ -59,6 +59,7 @@ import PamController.command.MultiportController; import PamController.command.NetworkController; import PamController.command.TerminalController; import PamController.command.WatchdogComms; +import PamController.fileprocessing.ReprocessManager; import PamController.masterReference.MasterReferencePoint; import PamController.settings.output.xml.PamguardXMLWriter; import PamController.settings.output.xml.XMLWriterDialog; @@ -119,6 +120,7 @@ public class PamController implements PamControllerInterface, PamSettings { public static final int PAM_STALLED = 3; public static final int PAM_INITIALISING = 4; public static final int PAM_STOPPING = 5; + public static final int PAM_COMPLETE = 6; // status' for RunMode = RUN_PAMVIEW public static final int PAM_LOADINGDATA = 2; @@ -705,7 +707,7 @@ public class PamController implements PamControllerInterface, PamSettings { Platform.exit(); // terminate the JVM - System.exit(0); + System.exit(getPamStatus()); } /** @@ -1021,6 +1023,24 @@ public class PamController implements PamControllerInterface, PamSettings { } return foundUnits; } + + /** + * Get an Array list of PamControlledUnits of a particular class (exact matches only). + * @param unitClass PamControlledUnit class + * @return List of current instances of this class. + */ + public ArrayList findControlledUnits(Class unitClass, boolean includeSubClasses) { + if (includeSubClasses == false) { + return findControlledUnits(unitClass); + } + ArrayList foundUnits = new ArrayList<>(); + for (int i = 0; i < getNumControlledUnits(); i++) { + if (unitClass.isAssignableFrom(pamControlledUnits.get(i).getClass())) { + foundUnits.add(pamControlledUnits.get(i)); + } + } + return foundUnits; + } /** * Check whether a controlled unit exists based on it's name. @@ -1191,10 +1211,24 @@ public class PamController implements PamControllerInterface, PamSettings { return false; } - if (saveSettings) { - saveSettings(PamCalendar.getSessionStartTime()); - } + /* + * Now we do some extra checks on the stores to see if we want to overwite data, + * carry on from where we left off, etc. + */ + if (saveSettings && getRunMode() == RUN_NORMAL) { // only true on a button press or network start. + ReprocessManager reprocessManager = new ReprocessManager(); + boolean goonthen = reprocessManager.checkOutputDataStatus(); + if (goonthen == false) { + System.out.println("Data processing will not start since you've chosen not to overwrite existing output data"); + pamStop(); + return false; + } + if (saveSettings) { + saveSettings(PamCalendar.getSessionStartTime()); + } + } + StorageOptions.getInstance().setBlockOptions(); t1 = System.currentTimeMillis(); diff --git a/src/PamController/RawInputControlledUnit.java b/src/PamController/RawInputControlledUnit.java new file mode 100644 index 00000000..8ac362a0 --- /dev/null +++ b/src/PamController/RawInputControlledUnit.java @@ -0,0 +1,21 @@ +package PamController; + +public abstract class RawInputControlledUnit extends PamControlledUnit { + + + public static final int RAW_INPUT_UNKNOWN = 0; + public static final int RAW_INPUT_FILEARCHIVE = 1; + public static final int RAW_INPUT_REALTIME = 2; + + public RawInputControlledUnit(String unitType, String unitName) { + super(unitType, unitName); + } + + /** + * Type of data input, which can be one of RAW_INPUT_UNKNOWN (0), + * RAW_INPUT_FILEARCHIVE (1), or RAW_INPUT_REALTIME (2) + * @return + */ + public abstract int getRawInputType(); + +} diff --git a/src/PamController/fileprocessing/ReprocessChoiceDialog.java b/src/PamController/fileprocessing/ReprocessChoiceDialog.java new file mode 100644 index 00000000..f2b9eee5 --- /dev/null +++ b/src/PamController/fileprocessing/ReprocessChoiceDialog.java @@ -0,0 +1,116 @@ +package PamController.fileprocessing; + +import java.awt.BorderLayout; +import java.awt.GridBagConstraints; +import java.awt.GridBagLayout; +import java.awt.Window; +import java.util.List; + +import javax.swing.ButtonGroup; +import javax.swing.JLabel; +import javax.swing.JPanel; +import javax.swing.JRadioButton; +import javax.swing.border.TitledBorder; + +import PamUtils.PamCalendar; +import PamView.dialog.PamDialog; +import PamView.dialog.PamGridBagContraints; +import PamView.dialog.warn.WarnOnce; +import PamView.panel.PamAlignmentPanel; + +public class ReprocessChoiceDialog extends PamDialog { + + private static final long serialVersionUID = 1L; + + private static ReprocessChoiceDialog singleInstance; + + private JRadioButton[] choiceButtons; + + private ReprocessStoreChoice chosenChoice = null; + + private StoreChoiceSummary choiceSummary; + + private ReprocessChoiceDialog(Window parentFrame, StoreChoiceSummary choiceSummary) { + super(parentFrame, "Existing Output Data", false); + this.choiceSummary = choiceSummary; + JPanel mainPanel = new JPanel(new BorderLayout()); + JPanel infoPanel = new JPanel(new GridBagLayout()); + infoPanel.setBorder(new TitledBorder("Data Summary")); + mainPanel.add(infoPanel, BorderLayout.NORTH); + GridBagConstraints c = new PamGridBagContraints(); + c.gridx = c.gridy = 0; + String inStr = String.format("Input data dates: %s to %s", PamCalendar.formatDBDateTime(choiceSummary.getInputStartTime()), + PamCalendar.formatDBDateTime(choiceSummary.getInputEndTime())); + infoPanel.add(new JLabel(inStr), c); + c.gridy++; + String outStr = String.format("Output data dates: %s to %s", PamCalendar.formatDBDateTime(choiceSummary.getOutputStartTime()), + PamCalendar.formatDBDateTime(choiceSummary.getOutputEndTime())); + infoPanel.add(new JLabel(outStr), c); + String stateStr; + if (choiceSummary.isProcessingComplete()) { + stateStr = "Processing appears to be complete"; + } + else { + stateStr = "Processing appears to be partially complete"; + } + c.gridy++; + infoPanel.add(new JLabel(stateStr), c); + + + JPanel choicePanel = new PamAlignmentPanel(new GridBagLayout(), BorderLayout.WEST); + choicePanel.setBorder(new TitledBorder("Chose what to do")); + c = new PamGridBagContraints(); + mainPanel.add(BorderLayout.SOUTH, choicePanel); + List userChoices = choiceSummary.getChoices(); + choiceButtons = new JRadioButton[userChoices.size()]; + ButtonGroup bg = new ButtonGroup(); + for (int i = 0; i < userChoices.size(); i++) { + ReprocessStoreChoice aChoice = userChoices.get(i); + choiceButtons[i] = new JRadioButton(aChoice.toString()); + choiceButtons[i].setToolTipText(aChoice.getToolTip()); + choicePanel.add(choiceButtons[i], c); + c.gridy++; + } + setDialogComponent(mainPanel); + getCancelButton().setVisible(false); + } + + public static ReprocessStoreChoice showDialog(Window parentFrame, StoreChoiceSummary choices) { +// if (singleInstance == null || singleInstance.getOwner() != parentFrame) { + singleInstance = new ReprocessChoiceDialog(parentFrame, choices); +// } + singleInstance.setVisible(true); + return singleInstance.chosenChoice; + } + + @Override + public boolean getParams() { + List userChoices = choiceSummary.getChoices(); + for (int i = 0; i < choiceButtons.length; i++) { + if (choiceButtons[i].isSelected()) { + chosenChoice = userChoices.get(i); + break; + } + } + if (chosenChoice == ReprocessStoreChoice.OVERWRITEALL) { + String w = "Are you sure you want to delete / overwrite all existing output data ?"; + int ans = WarnOnce.showWarning("Overwrite existing data", w, WarnOnce.OK_CANCEL_OPTION); + if (ans == WarnOnce.CANCEL_OPTION) { + return false; + } + } + return chosenChoice != null; + } + + @Override + public void cancelButtonPressed() { + chosenChoice = null; + } + + @Override + public void restoreDefaultSettings() { + // TODO Auto-generated method stub + + } + +} diff --git a/src/PamController/fileprocessing/ReprocessManager.java b/src/PamController/fileprocessing/ReprocessManager.java new file mode 100644 index 00000000..94323f67 --- /dev/null +++ b/src/PamController/fileprocessing/ReprocessManager.java @@ -0,0 +1,273 @@ +package PamController.fileprocessing; + +import java.util.ArrayList; + +import PamController.DataInputStore; +import PamController.DataOutputStore; +import PamController.InputStoreInfo; +import PamController.OfflineDataStore; +import PamController.PamControlledUnit; +import PamController.PamController; +import PamController.PamGUIManager; +import PamController.RawInputControlledUnit; +import PamUtils.PamCalendar; +import PamView.dialog.warn.WarnOnce; +import pamguard.GlobalArguments; + +/** + * Set of functions to help decide what to do when reprocessing. + * These are probably all called from AcquisitionProcess, but it's easier to have them in their own class. + * @author dg50 + * + */ +public class ReprocessManager { + + /** + public ReprocessManager() { + // TODO Auto-generated constructor stub + } + + /* + * run checks on the output data storage system. If data already exist in the output + * we may not want to start again. + */ + public boolean checkOutputDataStatus() { + + StoreChoiceSummary choiceSummary = null; + if (isOfflineFiles()) { + choiceSummary = checkIOFilesStatus(); + } + else { + /* + * don't really need to do anything for real time processing since adding + * more data to existing stores is normal behaviour. + */ + return true; + } + if (choiceSummary == null) { + return true; + } + + // need to decide what to do based on the list of possible choices. + ReprocessStoreChoice choice = chosePartStoreAction(choiceSummary); + + if (choice == ReprocessStoreChoice.DONTSSTART) { + return false; + } + + boolean deleteOK = deleteOldData(choiceSummary, choice); + boolean setupOK = setupInputStream(choiceSummary, choice); + + return true; + + } + + + private boolean setupInputStream(StoreChoiceSummary choiceSummary, ReprocessStoreChoice choice) { + // work out the first file index and send it to the appropriate input module. + long deleteFrom = getDeleteFromTime(choiceSummary, choice); + ArrayList inputStores = PamController.getInstance().findControlledUnits(DataInputStore.class, true); + if (inputStores == null || inputStores.size() == 0) { + return false; + } + InputStoreInfo inputInfo = null; + boolean OK = true; + for (PamControlledUnit aPCU : inputStores) { + DataInputStore inputStore = (DataInputStore) aPCU; + OK &= inputStore.setAnalysisStartTime(deleteFrom); +// System.out.println("Input store info: " + inputInfo); + } + return OK; + } + + + /** + * Just gets on and does it. The user should already have been asked what they + * want to do, so don't ask again. + * @param choiceSummary + * @param choice + */ + private boolean deleteOldData(StoreChoiceSummary choiceSummary, ReprocessStoreChoice choice) { + long deleteFrom = getDeleteFromTime(choiceSummary, choice); + // go through the data stores and tell them to delete from that time. + if (deleteFrom == Long.MAX_VALUE) { + return false; + } + ArrayList outputStores = PamController.getInstance().findControlledUnits(DataOutputStore.class, true); + boolean partStores = false; + boolean ok = true; + for (PamControlledUnit aPCU : outputStores) { + DataOutputStore offlineStore = (DataOutputStore) aPCU; + ok &= offlineStore.deleteDataFrom(deleteFrom); + } + return ok; + } + + + private long getDeleteFromTime(StoreChoiceSummary choiceSummary, ReprocessStoreChoice choice) { + if (choice == null) { + return Long.MAX_VALUE; // I don't think this can happen, but you never know. + } + Long t = null; + switch (choice) { + case CONTINUECURRENTFILE: + t = choiceSummary.getInputTimeForIndex(choiceSummary.getFileIndexBefore(choiceSummary.getOutputEndTime())); + break; + case CONTINUENEXTFILE: + t = choiceSummary.getInputTimeForIndex(choiceSummary.getFileIndexAfter(choiceSummary.getOutputEndTime())); + break; + case DONTSSTART: // we should'nt get here with this option. + return Long.MAX_VALUE; + case OVERWRITEALL: + return 0; // delete from start. ( + case STARTNORMAL: // we should'nt get here with this option. + return Long.MAX_VALUE; + default: + break; + + } + if (t == null) { + // shouldn't happen, don't do any deleteing + return Long.MAX_VALUE; + } + else { + return t; + } + } + + + /** + * Check the output of current files and databases and return a flag to PamController saying whether or + * not processing should actually start, possibly overwriting, or if we need to not start to avoid overwriting. + * @return true if processing should start. + */ + private StoreChoiceSummary checkIOFilesStatus() { + /** + * Get information about the input. + * + */ + ArrayList inputStores = PamController.getInstance().findControlledUnits(DataInputStore.class, true); + if (inputStores == null || inputStores.size() == 0) { + return new StoreChoiceSummary(null, ReprocessStoreChoice.STARTNORMAL); + } + InputStoreInfo inputInfo = null; + for (PamControlledUnit aPCU : inputStores) { + DataInputStore inputStore = (DataInputStore) aPCU; + inputInfo = inputStore.getStoreInfo(true); +// System.out.println("Input store info: " + inputInfo); + } + StoreChoiceSummary choiceSummary = new StoreChoiceSummary(inputInfo); + + if (inputInfo == null || inputInfo.getFileStartTimes() == null) { + choiceSummary.addChoice(ReprocessStoreChoice.STARTNORMAL); + return choiceSummary; + } + + ArrayList outputStores = PamController.getInstance().findControlledUnits(DataOutputStore.class, true); + boolean partStores = false; + for (PamControlledUnit aPCU : outputStores) { + DataOutputStore offlineStore = (DataOutputStore) aPCU; + StoreStatus status = offlineStore.getStoreStatus(false); + if (status == null) { + continue; + } + if (status.getStoreStatus() == StoreStatus.STATUS_HASDATA) { + status = offlineStore.getStoreStatus(true); // get more detail. + partStores = true; + System.out.printf("Storage %s already contains some data\n", offlineStore.getDataSourceName()); + choiceSummary.testOutputEndTime(status.getLastDataTime()); + choiceSummary.testOutputStartTime(status.getFirstDataTime()); + } + } + + if (partStores == false) { + choiceSummary.addChoice(ReprocessStoreChoice.STARTNORMAL); + return choiceSummary; + } + if (choiceSummary.getInputStartTime() >= choiceSummary.getOutputEndTime()) { + /* + * looks like it's new data that starts after the end of the current store, + * so there is no need to do anything. + */ + choiceSummary.addChoice(ReprocessStoreChoice.STARTNORMAL); + return choiceSummary; + } + /* + * If we land here, it looks like we have overlapping data. so need to make a decision + * First, check to see if processing has actually completed which will be the case if + * the data time and the end of the files are the same. + */ + choiceSummary.addChoice(ReprocessStoreChoice.DONTSSTART); + choiceSummary.addChoice(ReprocessStoreChoice.OVERWRITEALL); + if (choiceSummary.isProcessingComplete() == false) { + choiceSummary.addChoice(ReprocessStoreChoice.CONTINUECURRENTFILE); + choiceSummary.addChoice(ReprocessStoreChoice.CONTINUENEXTFILE); + } + + return choiceSummary; + + } + + /** + * Either opens a dialog to ask the user, or uses a choice entered into the command line for nogui mode. + * Decide what to do with stores that already have data. Can return continue from end or overwrite + * in which case stores will be deleted and we'll start again. The chosen action will need to be + * communicated to the various inputs. + * @param choices + */ + private ReprocessStoreChoice chosePartStoreAction(StoreChoiceSummary choices) { + /** + * Do we really have to deal with multiple inputs ? Can I envisage a situation where there is + * ever more than one input going at any one time ? not really, but should I add code + * to make sure that there really can be only one ? i.e. two daq's would be allowed for real + * time processing, but only one for offline ? could do all I guess by looking at sources of + * all output data blocks and doing it on a case by case basis. All we have to do here though + * is to get an answer about what to do. + */ + // see if we've got a global parameter passed in as an argument + String arg = GlobalArguments.getParam(ReprocessStoreChoice.paramName); + if (arg != null) { + ReprocessStoreChoice choice = ReprocessStoreChoice.valueOf(arg); + if (choice == null) { + String warn = String.format("Reprocessing storage input parameter %s value \"%s\" is not a recognised value", ReprocessStoreChoice.paramName, arg); + WarnOnce.showWarning("Invalid input parameter", warn, WarnOnce.WARNING_MESSAGE); + } + if (choice == ReprocessStoreChoice.CONTINUECURRENTFILE || choice == ReprocessStoreChoice.CONTINUENEXTFILE) { + if (choices.isProcessingComplete()) { + return ReprocessStoreChoice.DONTSSTART; + } + } + return choice; + } + if (PamGUIManager.getGUIType() == PamGUIManager.NOGUI) { + System.out.println("In Nogui mode you should set a choice as to how to handle existing storage overwrites. Using default of overwriting everything"); + return ReprocessStoreChoice.OVERWRITEALL; + } + + // otherwise we'll need to show a dialog to let the user decide what to do + ReprocessStoreChoice choice = ReprocessChoiceDialog.showDialog(PamController.getMainFrame(), choices); + + return choice; + } + + /** + * Return true if we seem to be reprocessing offline files. + * Note that this might be the Tritech data as well as the sound acquisition so + * have added an abstract intermediate class on the controlled units so we can check them all. + * @return + */ + public boolean isOfflineFiles() { + ArrayList sources = PamController.getInstance().findControlledUnits(RawInputControlledUnit.class, true); + if (sources == null) { + return false; + } + for (PamControlledUnit pcu : sources) { + RawInputControlledUnit rawPCU = (RawInputControlledUnit) pcu; + if (rawPCU.getRawInputType() == RawInputControlledUnit.RAW_INPUT_FILEARCHIVE) { + return true; + } + } + return false; + } + +} diff --git a/src/PamController/fileprocessing/ReprocessStoreChoice.java b/src/PamController/fileprocessing/ReprocessStoreChoice.java new file mode 100644 index 00000000..0a6835db --- /dev/null +++ b/src/PamController/fileprocessing/ReprocessStoreChoice.java @@ -0,0 +1,53 @@ +package PamController.fileprocessing; + +/** + * Choices on what to do when re-processing data and finding that output data already exist. + * @author dg50 + * + */ +public enum ReprocessStoreChoice { + + + STARTNORMAL, CONTINUECURRENTFILE, CONTINUENEXTFILE, OVERWRITEALL, DONTSSTART; + + public static final String paramName = "-reprocessoption"; + + @Override + public String toString() { + switch (this) { + case STARTNORMAL: + return "Start normally. No risk of overwriting"; + case CONTINUECURRENTFILE: + return "Continue from start of last input file processed"; + case CONTINUENEXTFILE: + return "Continue from start of next input file to process"; + case DONTSSTART: + return "Don't start processing"; + case OVERWRITEALL: + return "Overwrite existing output data"; + default: + break; + } + return null; + } + + public String getToolTip() { + switch (this) { + case STARTNORMAL: + return "No risk of data overlap, so system will start normally"; + case CONTINUECURRENTFILE: + return "System will work out how far data processing has got and continue from the start of the file it stopped in"; + case CONTINUENEXTFILE: + return "System will work out how far data processing has got and continue from the start of the file AFTER the one it stopped in"; + case DONTSSTART: + return "Processing will not start. Select alternative storage locations / databases and try again"; + case OVERWRITEALL: + return "Overwrite existing output data. Existing data will be deleted"; + default: + break; + } + return null; + } + + +} diff --git a/src/PamController/fileprocessing/StoreChoiceSummary.java b/src/PamController/fileprocessing/StoreChoiceSummary.java new file mode 100644 index 00000000..f0f13c3a --- /dev/null +++ b/src/PamController/fileprocessing/StoreChoiceSummary.java @@ -0,0 +1,234 @@ +package PamController.fileprocessing; + +import java.util.ArrayList; +import java.util.List; + +import PamController.InputStoreInfo; + +/** + * Summary information about the data stores. + * @author dg50 + * + */ +public class StoreChoiceSummary { + + private long outputEndTime; + + private long outputStartTime; + + private List choices = new ArrayList<>(); + + private InputStoreInfo inputStoreInfo; + + public StoreChoiceSummary(InputStoreInfo info, ReprocessStoreChoice singleChoice) { + this.inputStoreInfo = info; + addChoice(singleChoice); + } + + public StoreChoiceSummary(long outputEndTime, InputStoreInfo inputStoreInfo) { + super(); + this.outputEndTime = outputEndTime; + this.inputStoreInfo = inputStoreInfo; + } + + public StoreChoiceSummary(InputStoreInfo inputInfo) { + this.inputStoreInfo = inputInfo; + } + + /** + * Get the number of choices. If it's only one, then there + * isn't a lot to do. If it's >1, then need a decision in the + * form of a command line instruction or a user dialog. + * @return number of choices. + */ + public int getNumChoices() { + return choices.size(); + } + + /** + * Is processing complete, i.e. last time in output matches last time + * in input data. + * @return true if processing appears to be complete. + */ + public boolean isProcessingComplete() { + if (inputStoreInfo == null) { + return false; + } + long inputEnd = getInputEndTime(); + long outputEnd = getOutputEndTime(); + long diff = inputEnd-outputEnd; + return (diff < 1000); + } + + /** + * Add a reasonable choice to what the user can select to do. + * @param choice + */ + public void addChoice(ReprocessStoreChoice choice) { + choices.add(choice); + } + + /** + * @return the start time of the first input file + */ + public Long getInputStartTime() { + if (inputStoreInfo == null) { + return null; + } + return inputStoreInfo.getFirstFileStart(); + } + + /** + * @return the start time of the first input file + */ + public Long getInputEndTime() { + if (inputStoreInfo == null) { + return null; + } + return inputStoreInfo.getLastFileEnd(); + } + + /** + * @return the outputEndTime + */ + public long getOutputEndTime() { + return outputEndTime; + } + + + /** + * Set the last data time, but only if the passed value + * is not null and is bigger than the current value. + * @param lastDataTime + * @return largest of current and passed value. + */ + public long testOutputEndTime(Long lastDataTime) { + if (lastDataTime == null) { + return this.getOutputEndTime(); + } + setOutputEndTime(Math.max(outputEndTime, lastDataTime)); + return getOutputEndTime(); + } + + /** + * Set the last data time, but only if the passed value + * is not null and is bigger than the current value. + * @param lastDataTime + * @return largest of current and passed value. + */ + public long testOutputStartTime(Long firstDataTime) { + if (firstDataTime == null) { + return this.getOutputStartTime(); + } + if (outputStartTime == 0 || firstDataTime < outputStartTime) { + outputStartTime = firstDataTime; + } + return getOutputStartTime(); + } + + /** + * @param outputEndTime the outputEndTime to set + */ + public void setOutputEndTime(long outputEndTime) { + this.outputEndTime = outputEndTime; + } + + /** + * @return the inputStoreInfo + */ + public InputStoreInfo getInputStoreInfo() { + return inputStoreInfo; + } + + /** + * @param inputStoreInfo the inputStoreInfo to set + */ + public void setInputStoreInfo(InputStoreInfo inputStoreInfo) { + this.inputStoreInfo = inputStoreInfo; + } + + /** + * @return the choices + */ + public List getChoices() { + return choices; + } + + /** + * @return the outputStartTime + */ + public long getOutputStartTime() { + return outputStartTime; + } + + /** + * @param outputStartTime the outputStartTime to set + */ + public void setOutputStartTime(long outputStartTime) { + this.outputStartTime = outputStartTime; + } + + /** + * Get the index of the file that starts before or exactly at the given time. + * @param inputEndTime + * @return index of file, or -1 if none found. + */ + public int getFileIndexBefore(Long inputEndTime) { + if (inputStoreInfo == null) { + return -1; + } + long[] fileStarts = inputStoreInfo.getFileStartTimes(); + if (fileStarts == null) { + return -1; + } + for (int i = fileStarts.length-1; i>= 0; i--) { + if (fileStarts[i] <= inputEndTime) { + return i; + } + } + return -1; + } + + /** + * Get the start time in millis of a file for the given index. + * @param fileIndex + * @return file time, or null if no file available. + */ + public Long getInputTimeForIndex(int fileIndex) { + if (inputStoreInfo == null) { + return null; + } + long[] fileStarts = inputStoreInfo.getFileStartTimes(); + if (fileStarts == null) { + return null; + } + if (fileIndex < 0 || fileIndex >= fileStarts.length) { + return null; + } + return fileStarts[fileIndex]; + } + + /** + * Get the index of the file that starts after the given time. + * @param inputEndTime + * @return index of file, or -1 if none found. + */ + public int getFileIndexAfter(Long inputEndTime) { + if (inputStoreInfo == null) { + return -1; + } + long[] fileStarts = inputStoreInfo.getFileStartTimes(); + if (fileStarts == null) { + return -1; + } + for (int i = 0; i < fileStarts.length; i++) { + if (fileStarts[i] > inputEndTime) { + return i; + } + } + return -1; + } + + + +} diff --git a/src/PamController/fileprocessing/StoreStatus.java b/src/PamController/fileprocessing/StoreStatus.java new file mode 100644 index 00000000..5f50fbc1 --- /dev/null +++ b/src/PamController/fileprocessing/StoreStatus.java @@ -0,0 +1,115 @@ +package PamController.fileprocessing; + +import java.io.File; + +import PamController.OfflineDataStore; + +/** + * Class to carry information about an OfflineDataStore. Used when restarting offline + * processing to help work out if we should overwrite, start again, etc. + * @author dg50 + * + */ +abstract public class StoreStatus { + + public static final int STATUS_MISSING = 1; + + public static final int STATUS_EMPTY = 2; + + public static final int STATUS_HASDATA = 3; + + private OfflineDataStore offlineDataStore; + + /** + * Time of first data, may be null if detail not asked for or if + * hasData is false. + */ + private Long firstDataTime; + + /** + * Time of last data, may be null if detail not asked for or if + * hasData is false. + */ + private Long lastDataTime; + + /** + * General status flag. + */ + private int storeStatus; + + public StoreStatus(OfflineDataStore offlineDataStore) { + this.offlineDataStore = offlineDataStore; + } + + /** + * Get the amount of free space for this storage. + * @return free space in bytes. + */ + public abstract long getFreeSpace(); + + public long getFreeSpace(String currDir) { + if (currDir == null) { + return 0; + } + File dirFile = new File(currDir); + long space = 0; + try { + space = dirFile.getUsableSpace(); + } + catch (SecurityException e) { + System.out.printf("Security exception getting space for %s: \n%s\n", currDir, e.getMessage()); + } + return space; + } + + /** + * @return the firstDataTime + */ + public Long getFirstDataTime() { + return firstDataTime; + } + + /** + * @param firstDataTime the firstDataTime to set + */ + public void setFirstDataTime(Long firstDataTime) { + this.firstDataTime = firstDataTime; + } + + /** + * @return the lastDataTime + */ + public Long getLastDataTime() { + return lastDataTime; + } + + /** + * @param lastDataTime the lastDataTime to set + */ + public void setLastDataTime(Long lastDataTime) { + this.lastDataTime = lastDataTime; + } + + /** + * @return the storeStatus + */ + public int getStoreStatus() { + return storeStatus; + } + + /** + * @param storeStatus the storeStatus to set + */ + public void setStoreStatus(int storeStatus) { + this.storeStatus = storeStatus; + } + + /** + * @return the offlineDataStore + */ + public OfflineDataStore getOfflineDataStore() { + return offlineDataStore; + } + + +} diff --git a/src/PamUtils/worker/filelist/WavFileType.java b/src/PamUtils/worker/filelist/WavFileType.java index a00aa420..25300245 100644 --- a/src/PamUtils/worker/filelist/WavFileType.java +++ b/src/PamUtils/worker/filelist/WavFileType.java @@ -55,9 +55,19 @@ public class WavFileType extends File { * @return the audioInfo */ public AudioFormat getAudioInfo() { + if (audioInfo == null) { + audioInfo = getAudioFormat(); + } return audioInfo; } - + + /** + * Get the audio format. + * @return the audio format. + */ + private AudioFormat getAudioFormat() { + return getAudioFormat(this); + } /** * Get the audio format. diff --git a/src/PamguardMVC/uid/binaryUIDFunctions.java b/src/PamguardMVC/uid/binaryUIDFunctions.java index f8959935..310f3b23 100644 --- a/src/PamguardMVC/uid/binaryUIDFunctions.java +++ b/src/PamguardMVC/uid/binaryUIDFunctions.java @@ -46,6 +46,7 @@ import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpression; import javax.xml.xpath.XPathFactory; +import org.apache.commons.io.comparator.NameFileComparator; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -391,7 +392,7 @@ public class binaryUIDFunctions { // System.out.println("Warning - no " + filePrefix + " binary files found in " + binStore.getBinaryStoreSettings().getStoreLocation()); return maxUID; } - Collections.sort(binFiles); + Collections.sort(binFiles, NameFileComparator.NAME_COMPARATOR); // loop through the binary files from the last one to the first, and stop as // soon as we find diff --git a/src/binaryFileStorage/BinaryStore.java b/src/binaryFileStorage/BinaryStore.java index c2007cb2..3f3ebb02 100644 --- a/src/binaryFileStorage/BinaryStore.java +++ b/src/binaryFileStorage/BinaryStore.java @@ -36,6 +36,8 @@ import dataGram.DatagramManager; import dataMap.OfflineDataMap; import dataMap.OfflineDataMapPoint; import PamController.AWTScheduler; +import PamController.DataInputStore; +import PamController.DataOutputStore; import PamController.OfflineDataStore; import PamController.PamControlledUnit; import PamController.PamControlledUnitGUI; @@ -49,6 +51,7 @@ import PamController.PamSettingsGroup; import PamController.PamSettingsSource; import PamController.StorageOptions; import PamController.StorageParameters; +import PamController.fileprocessing.StoreStatus; import PamController.status.ModuleStatus; import PamController.status.QuickRemedialAction; import PamModel.SMRUEnable; @@ -89,7 +92,7 @@ import binaryFileStorage.layoutFX.BinaryStoreGUIFX; * */ public class BinaryStore extends PamControlledUnit implements PamSettings, -PamSettingsSource, OfflineDataStore { +PamSettingsSource, DataOutputStore { public static final String fileType = "pgdf"; @@ -1113,6 +1116,8 @@ PamSettingsSource, OfflineDataStore { String lastFailedStream = null; + private BinaryStoreStatusFuncs binaryStoreStatusFuncs; + public boolean removeMapPoint(File aFile, ArrayList streams) { BinaryHeaderAndFooter bhf = getFileHeaderAndFooter(aFile); if (bhf == null || bhf.binaryHeader == null) { @@ -1354,6 +1359,26 @@ PamSettingsSource, OfflineDataStore { } return newFile; } + + /** + * Find the noise file to match a given data file. + * @param dataFile data file. + * @param checkExists check the file exists and if it doens't return null + * @return index file to go with the data file. + */ + public File findNoiseFile(File dataFile, boolean checkExists) { +// String filePath = dataFile.getAbsolutePath(); +// // check that the last 4 characters are "pgdf" +// int pathLen = filePath.length(); +// String newPath = filePath.substring(0, pathLen-4) + indexFileType; + File newFile = swapFileType(dataFile, noiseFileType); + if (checkExists) { + if (newFile.exists() == false) { + return null; + } + } + return newFile; + } /** * Create an index file (pgdx) name from a data file (pgdf) file name @@ -1488,7 +1513,7 @@ PamSettingsSource, OfflineDataStore { * @param folder folder to search * @param filter file filter */ - private void listDataFiles(ArrayList fileList, File folder, PamFileFilter filter) { + public void listDataFiles(ArrayList fileList, File folder, PamFileFilter filter) { File[] newFiles = folder.listFiles(filter); if (newFiles == null) { return; @@ -1684,7 +1709,7 @@ PamSettingsSource, OfflineDataStore { * @param binaryObjectData * @param dataSink */ - private void unpackAnnotationData(int fileVersion, PamDataUnit createdUnit, BinaryObjectData binaryObjectData, BinaryDataSink dataSink) { + protected void unpackAnnotationData(int fileVersion, PamDataUnit createdUnit, BinaryObjectData binaryObjectData, BinaryDataSink dataSink) { //System.out.println("Hello annotation " + binaryObjectData.getAnnotationDataLength()); if (binaryObjectData.getAnnotationDataLength() == 0) { @@ -2211,7 +2236,7 @@ PamSettingsSource, OfflineDataStore { - private boolean reportError(String string) { + boolean reportError(String string) { System.out.println(string); return false; } @@ -2509,4 +2534,17 @@ PamSettingsSource, OfflineDataStore { return binaryStoreSettings.getNoiseStoreType(); } + @Override + public StoreStatus getStoreStatus(boolean getDetail) { + if (binaryStoreStatusFuncs == null) { + binaryStoreStatusFuncs = new BinaryStoreStatusFuncs(this); + } + return binaryStoreStatusFuncs.getStoreStatus(getDetail); + } + @Override + public boolean deleteDataFrom(long timeMillis) { + BinaryStoreDeleter storeDeleter = new BinaryStoreDeleter(this); + return storeDeleter.deleteDataFrom(timeMillis); + } + } diff --git a/src/binaryFileStorage/BinaryStoreDeleter.java b/src/binaryFileStorage/BinaryStoreDeleter.java new file mode 100644 index 00000000..1497ec45 --- /dev/null +++ b/src/binaryFileStorage/BinaryStoreDeleter.java @@ -0,0 +1,363 @@ +package binaryFileStorage; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.File; +import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.io.comparator.NameFileComparator; + +import PamUtils.PamCalendar; +import PamUtils.PamFileFilter; +import PamguardMVC.DataUnitBaseData; +import PamguardMVC.PamDataBlock; +import PamguardMVC.PamDataUnit; + +public class BinaryStoreDeleter { + + private static final int FILE_DELETE_ERROR = 1; + private static final int FILE_TOO_EARLY = 2; + private static final int FILE_DELETED = 3; + private static final int FILE_PARTIAL_DELETE = 4; + + + private BinaryStore binaryStore; + + private FileFilter directoryFilter; + + private BinaryStoreStatusFuncs binaryStoreStatusFuncs; + + public BinaryStoreDeleter(BinaryStore binaryStore) { + this.binaryStore = binaryStore; + directoryFilter = new DirectoryFilter(); + binaryStoreStatusFuncs = new BinaryStoreStatusFuncs(binaryStore); + } + + public boolean deleteDataFrom(long timeMillis) { + if (timeMillis == 0) { + return deleteEverything(); + } + else { + return deleteFrom(timeMillis); + } + } + + private class DirectoryFilter implements java.io.FileFilter { + + @Override + public boolean accept(File pathname) { + return pathname.isDirectory(); + } + + } + + private boolean deleteEverything() { + ArrayList fileList = new ArrayList(); + String root = binaryStore.binaryStoreSettings.getStoreLocation(); + if (root == null) { + return false; + } + File rootFolder = new File(root); + PamFileFilter binaryDataFilter = new PamFileFilter("Binary Data Files", BinaryStore.fileType); + binaryDataFilter.addFileType(BinaryStore.indexFileType); + binaryDataFilter.addFileType(BinaryStore.noiseFileType); + binaryDataFilter.setAcceptFolders(true); + + binaryStore.listDataFiles(fileList, rootFolder, binaryDataFilter); + int errors = 0; + for (File aFile : fileList) { + try { + aFile.delete(); + } + catch (Exception e) { + errors++; + } + } + deleteEmptyFolders(); + return errors == 0; + } + + private boolean deleteFrom(long timeMillis) { + /* + * need to go through the data one stream at a time so that + * we can pick files off from the end of the list. + */ + ArrayList streams = BinaryStore.getStreamingDataBlocks(true); + int errors = 0; + for (PamDataBlock aBlock : streams) { + boolean ok = deleteFrom(aBlock, timeMillis); + if (!ok) { + errors++; + } + } + + deleteEmptyFolders(); + return false; + } + + private boolean deleteFrom(PamDataBlock aBlock, long timeMillis) { + System.out.printf("Deleting binary data for %s from %s\n", aBlock.getDataName(), PamCalendar.formatDBDateTime(timeMillis)); + BinaryDataSource dataSource = aBlock.getBinaryDataSource(); + if (dataSource == null) { + return true; // don't see how this can happen. + } + // first deal with pgdf and pgdx files, then noise. + String filePrefix = dataSource.createFilenamePrefix(); + List binFiles = binaryStore.listAllFilesWithPrefix(filePrefix); + if (binFiles == null || binFiles.isEmpty()) { + return true; // nothing to delete. + } + Collections.sort(binFiles, NameFileComparator.NAME_COMPARATOR); + for (int i = binFiles.size()-1; i >= 0; i--) { + int ans = deleteFileFrom(aBlock, binFiles.get(i), timeMillis); + if (ans == FILE_TOO_EARLY) { + break; + } + } + + return true; + } + + /** + * Delete a specific file from a specific time. If the start of the file + * is after timeMillis, delete the entire file, otherwise it will have + * to be a partial delete. + * @param aBlock + * @param file + * @param timeMillis + * @return + */ + private int deleteFileFrom(PamDataBlock aBlock, File dataFile, long timeMillis) { + File indexFile = binaryStore.findIndexFile(dataFile, true); + if (indexFile == null) { + indexFile = dataFile; + } + File noiseFile = binaryStore.findNoiseFile(dataFile, true); + // get the header. + boolean headOk = false; + BinaryHeader binaryHead = new BinaryHeader(); + try { + DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(indexFile))); + headOk = binaryHead.readHeader(dis); + dis.close(); + } + catch (IOException e) { + headOk = false; + } + if (headOk == false || binaryHead.getDataDate() >= timeMillis) { + boolean deleteOk = deleteFileSet(dataFile); + return deleteOk ? FILE_DELETED : FILE_DELETE_ERROR; + } + /** + * Now need to see if the file is earlier than we want, in which case we return + * immediately and won't look at any more files. + */ + BinaryFooter fileEnd = binaryStoreStatusFuncs.findLastData(dataFile); + if (fileEnd == null) { + // the file has no footer and no data, so must be corrupt, so delete it. + boolean deleteOk = deleteFileSet(dataFile); + return deleteOk ? FILE_DELETED : FILE_DELETE_ERROR; + } + if (fileEnd.getDataDate() <= timeMillis) { + /* + * this file is earlier than our delete time, so we don't want to delete it + * and need to send a message saying not to delete anything else either. + */ + return FILE_TOO_EARLY; + } + /** + * If we land here, it looks like we're in the realm of needing to partially delete + * a file / set of data and noise files. What a pain ! Will need to do + * the deleting and update the index file. f** knows what to do about a + * serialized datamap. + */ + partialCopyFile(aBlock, dataFile, timeMillis); + if (indexFile != null) { + partialCopyFile(aBlock, indexFile, timeMillis); + } + if (noiseFile != null) { + partialCopyFile(aBlock, noiseFile, timeMillis); + } + + return FILE_PARTIAL_DELETE; + } + + private boolean partialCopyFile(PamDataBlock aBlock, File dataFile, long timeMillis) { + try { + BinaryInputStream inputStream = new BinaryInputStream(binaryStore, aBlock); + if (inputStream.openFile(dataFile) == false) { + return false; + } + + BinaryDataSource dataSource = aBlock.getBinaryDataSource(); + + File tempFile = new File(dataFile.getAbsolutePath() + ".tmp"); + BinaryOutputStream outputStream = new BinaryOutputStream(binaryStore, aBlock); + dataSource.setBinaryStorageStream(outputStream); + + BinaryObjectData binaryObjectData; + BinaryHeader bh = inputStream.readHeader(); + if (bh==null) { + return false; + } + outputStream.writeHeader(bh.getDataDate(), bh.getAnalysisDate()); + ModuleHeader mh = null; + + BinaryFooter bf = null; + int inputFormat = bh.getHeaderFormat(); + while ((binaryObjectData = inputStream.readNextObject(inputFormat)) != null) { + + switch (binaryObjectData.getObjectType()) { + case BinaryTypes.FILE_FOOTER: + // this is unlikely to happen, since we'll probably already have found an index file. + bf = new BinaryFooter(); + bf.readFooterData(binaryObjectData.getDataInputStream(), inputFormat); + bf.setDataDate(timeMillis); + outputStream.writeFileFooter(bf); + break; + case BinaryTypes.MODULE_HEADER: + mh = dataSource.sinkModuleHeader(binaryObjectData, bh); + outputStream.writeModuleHeader(); + break; + case BinaryTypes.MODULE_FOOTER: + ModuleFooter mf = dataSource.sinkModuleFooter(binaryObjectData, bh, mh); + outputStream.writeModuleFooter(); + break; + case BinaryTypes.DATAGRAM: +// dataSource. + break; + default: // should be data. + DataUnitBaseData baseData = binaryObjectData.getDataUnitBaseData(); + if (baseData == null) { + continue; + } + if (baseData.getTimeMilliseconds() > timeMillis) { + continue; + } + /* + * otherwise we need to store this data unit. I think we can just copy in the + * existing binary data to the new file non ? Might mess the datagram slightly, + * but that is only in the index file and can sort itself out. + * better to make a data unit and then rewrite it I think. + */ + PamDataUnit dataUnit = dataSource.sinkData(binaryObjectData, bh, inputFormat); + if (dataUnit != null) { + dataUnit.getBasicData().mergeBaseData(binaryObjectData.getDataUnitBaseData()); + binaryStore.unpackAnnotationData(bh.getHeaderFormat(), dataUnit, binaryObjectData, null); + dataSource.saveData(dataUnit); + } + + } + } + + outputStream.closeFile(); + inputStream.closeFile(); + + /* + * Now file final stage - copy the temp file in place of the + * original file. + */ + boolean deletedOld = false; + try { + deletedOld = dataFile.delete(); + } + catch (SecurityException e) { + System.out.println("Error deleting old pgdf file: " + dataFile.getAbsolutePath()); + e.printStackTrace(); + } + + boolean renamedNew = false; + try { + renamedNew = tempFile.renameTo(dataFile); + } + catch (SecurityException e) { + System.out.println("Error renaming new pgdf file: " + tempFile.getAbsolutePath() + + " to " + dataFile.getAbsolutePath()); + e.printStackTrace(); + } + if (renamedNew == false) { + if (deletedOld == false) { + binaryStore.reportError("Unable to delete " + dataFile.getAbsolutePath()); + } + return binaryStore.reportError(String.format("Unable to rename %s to %s", + tempFile.getAbsolutePath(), dataFile.getAbsolutePath())); + } + + return true; + + } + catch (Exception ex) { + return false; + } + + } + + /** + * Delete a set of files, including main data file, index file and noise file. + * @param dataFile + * @return + */ + private boolean deleteFileSet(File dataFile) { + try { + File indexFile = binaryStore.findIndexFile(dataFile, true); + File noiseFile = binaryStore.findNoiseFile(dataFile, true); + boolean deleteOk = true; + deleteOk &= dataFile.delete(); + if (indexFile != null) { + deleteOk &= indexFile.delete(); + } + if (noiseFile != null) { + deleteOk &= noiseFile.delete(); + } + return deleteOk; + } + catch (Exception e) { + return false; + } + + } + + private void deleteEmptyFolders() { + String root = binaryStore.binaryStoreSettings.getStoreLocation(); + if (root == null) { + return; + } + /** + * Iterate through the root folder first and then call a + * recursive function to delete sub folders. this will stop the + * root folder from being deleted, but sub folders will get deleted if + * they have no files (of any type) in them. + */ + File rootFolder = new File(root); + File[] subFolders = rootFolder.listFiles(directoryFilter); + if (subFolders == null) { + return; + } + for (int i = 0; i < subFolders.length; i++) { + deleteEmptyFolders(subFolders[i]); + } + } + + private void deleteEmptyFolders(File file) { + File[] subFolders = file.listFiles(directoryFilter); + for (int i = 0; i < subFolders.length; i++) { + deleteEmptyFolders(subFolders[i]); + } + // see if there is anything at all in this folder + File[] remaining = file.listFiles(); + if (remaining.length == 0) { + try { + file.delete(); + } + catch (Exception e) { + System.out.printf("Binary folder %s cannot be deleted: %s\n", file.getAbsolutePath(), e.getMessage()); + } + } + } + +} diff --git a/src/binaryFileStorage/BinaryStoreStatus.java b/src/binaryFileStorage/BinaryStoreStatus.java new file mode 100644 index 00000000..53c00717 --- /dev/null +++ b/src/binaryFileStorage/BinaryStoreStatus.java @@ -0,0 +1,166 @@ +package binaryFileStorage; + +import PamController.fileprocessing.StoreStatus; + +public class BinaryStoreStatus extends StoreStatus { + + private BinaryStore binaryStore; + + private BinaryHeader firstHeader; + + private BinaryFooter lastFooter; + + private BinaryFooter lastData; + + public BinaryStoreStatus(BinaryStore binaryStore) { + super(binaryStore); + this.binaryStore = binaryStore; + } + + public BinaryStoreStatus(BinaryStore binaryStore, BinaryHeader firstHead, BinaryFooter lastFoot, + BinaryFooter lastData) { + super(binaryStore); + this.binaryStore = binaryStore; + this.firstHeader = firstHead; + this.lastFooter = lastFoot; + this.lastData = lastData; + } + + + @Override + public Long getFirstDataTime() { + if (firstHeader != null) { + return firstHeader.getDataDate(); + } + return null; + } + + @Override + public Long getLastDataTime() { + if (lastData != null) { + return lastData.getDataDate(); + } + if (lastFooter != null) { + return lastFooter.getDataDate(); + } + return null; + } + + /** + * @return the firstHeader + */ + public BinaryHeader getFirstHeader() { + return firstHeader; + } + + /** + * @param firstHeader the firstHeader to set + */ + public void setFirstHeader(BinaryHeader firstHeader) { + this.firstHeader = firstHeader; + if (firstHeader != null) { + setFirstDataTime(firstHeader.getDataDate()); + } + else { + setFirstDataTime(null); + } + } + + /** + * @return the lastFooter + */ + public BinaryFooter getLastFooter() { + return lastFooter; + } + + /** + * @param lastFooter the lastFooter to set + */ + public void setLastFooter(BinaryFooter lastFooter) { + this.lastFooter = lastFooter; + } + + /** + * @return the lastData + */ + public BinaryFooter getLastData() { + return lastData; + } + + /** + * @param lastData the lastData to set + */ + public void setLastData(BinaryFooter lastData) { + this.lastData = lastData; + if (lastData != null) { + setLastDataTime(lastData.getDataDate()); + } + else { + setLastDataTime(null); + } + } + + @Override + public long getFreeSpace() { + return getFreeSpace(binaryStore.getBinaryStoreSettings().getStoreLocation()); + } + + /** + * Looking overall for first header, last footers, etc. + * @param blockStatus + */ + public void considerBlockStatus(BinaryStoreStatus blockStatus) { + considerFirstHeader(blockStatus.firstHeader); + considerLastFooter(blockStatus.lastFooter); + considerLastData(blockStatus.lastData); + } + + /** + * Take a footer for last data with the later date + * @param footer + */ + private void considerLastData(BinaryFooter footer) { + if (footer == null || footer.getDataDate() == 0) { + return; + } + if (lastData == null || lastData.getDataDate() == 0) { + lastData = footer; + } + if (footer.getDataDate() > lastData.getDataDate()) { + lastData = footer; + } + } + + /** + * Take a footer for last footer with the later date + * @param footer + */ + private void considerLastFooter(BinaryFooter footer) { + if (footer == null || footer.getDataDate() == 0) { + return; + } + if (lastFooter == null || lastFooter.getDataDate() == 0) { + lastFooter = footer; + } + if (footer.getDataDate() > lastFooter.getDataDate()) { + lastFooter = footer; + } + } + + /** + * Take a header for the first header with the earliest date. + * @param header + */ + private void considerFirstHeader(BinaryHeader header) { + if (header == null || header.getDataDate() == 0) { + return; + } + if (this.firstHeader == null || firstHeader.getDataDate() == 0) { + this.firstHeader = header; + } + if (header.getDataDate() < firstHeader.getDataDate()) { + firstHeader = header; + } + } + +} diff --git a/src/binaryFileStorage/BinaryStoreStatusFuncs.java b/src/binaryFileStorage/BinaryStoreStatusFuncs.java new file mode 100644 index 00000000..31d9e9dc --- /dev/null +++ b/src/binaryFileStorage/BinaryStoreStatusFuncs.java @@ -0,0 +1,270 @@ +package binaryFileStorage; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.io.comparator.NameFileComparator; + +import PamController.fileprocessing.StoreStatus; +import PamUtils.PamFileFilter; +import PamguardMVC.DataUnitBaseData; +import PamguardMVC.PamDataBlock; + +/** + * Set of functions used at restarts to determine the status of the binary store. + * @author dg50 + * + */ +public class BinaryStoreStatusFuncs { + + private BinaryStore binaryStore; + + public BinaryStoreStatusFuncs(BinaryStore binaryStore) { + this.binaryStore = binaryStore; + } + + public StoreStatus getStoreStatus(boolean getDetail) { + BinaryStoreStatus binStoreStatus = new BinaryStoreStatus(binaryStore); + binStoreStatus.setStoreStatus(checkStoreStatus()); + if (getDetail && binStoreStatus.getStoreStatus() == StoreStatus.STATUS_HASDATA) { + binStoreStatus = getStoreDetail(binStoreStatus); + } + return binStoreStatus; + } + + private BinaryStoreStatus getStoreDetail(BinaryStoreStatus binStoreStatus) { + // go through every stream and find it's first and last data times. + long lastTime = Long.MIN_VALUE; + long firstTime = Long.MAX_VALUE; + ArrayList streams = BinaryStore.getStreamingDataBlocks(true); + for (PamDataBlock aBlock : streams) { + BinaryDataSource dataSource = aBlock.getBinaryDataSource(); + if (dataSource == null) { + continue; + } + BinaryStoreStatus blockStatus = getStreamStartEnd(dataSource); + binStoreStatus.considerBlockStatus(blockStatus); + } + + return binStoreStatus; + } + + + private BinaryStoreStatus getStreamStartEnd(BinaryDataSource dataSource) { + String filePrefix = dataSource.createFilenamePrefix(); + List binFiles = binaryStore.listAllFilesWithPrefix(filePrefix); + if (binFiles == null || binFiles.isEmpty()) { + return null; + } + Collections.sort(binFiles, NameFileComparator.NAME_COMPARATOR); + BinaryHeader firstHead = findFirstHeader(binFiles); + BinaryFooter lastFoot = findLastFooter(binFiles); + BinaryFooter lastData = findLastData(binFiles); + BinaryStoreStatus storeStatus = new BinaryStoreStatus(binaryStore, firstHead, lastFoot, lastData); + return storeStatus; + } + + /** + * Get the last footer. This may be in the last file, but may not be if things + * crashed and the last file didn't get completed, i nwhich case it will be in + * the file before. + * @param binFiles + * @return + */ + private BinaryFooter findLastFooter(List binFiles) { + for (int i = binFiles.size()-1; i>=0; i--) { + File aFile = binFiles.get(i); + /* + * if the last file was completed correctly, it will have an index file. If there isn't + * an index file it's very unlikely there will be a footer in the main file + */ + File indexFile = binaryStore.findIndexFile(aFile, true); + if (indexFile == null) { + continue; + } + BinaryHeaderAndFooter headAndFoot = binaryStore.readHeaderAndFooter(indexFile); + if (headAndFoot != null && headAndFoot.binaryFooter != null) { + return headAndFoot.binaryFooter; + } + } + return null; + } + + /** + * Get the last time of any data, whether it's from a header, footer, or actual data. + * @param binFiles + * @return + */ + private BinaryFooter findLastData(List binFiles) { + for (int i = binFiles.size()-1; i>=0; i--) { + File aFile = binFiles.get(i); + BinaryFooter bf = findLastData(aFile); + if (bf != null) { + return bf; + } + } + return null; + } + + /** + * Get the last data in a file. Hopefully this comes + * from the footer, but it might have to look at all data if + * the footer is absent or the index file missing. + * @param aFile + * @return + */ + public BinaryFooter findLastData(File aFile) { + Long lastUID = null; + Long lastTime = null; + Long firstUID = null; + + File indexFile = binaryStore.findIndexFile(aFile, true); + if (indexFile != null) { + BinaryHeaderAndFooter headAndFoot = binaryStore.readHeaderAndFooter(indexFile); + if (headAndFoot != null && headAndFoot.binaryFooter != null) { + return headAndFoot.binaryFooter; + } + } + /* + * otherwise it would seem that we've a file without a valid end, so unpack it and + * get the UID and time of the last item in the file. Can return these in the form of + * a BinaryFooter since it's pretty much the same information needed. + */ + try { + // need to work through the file now. + BinaryInputStream inputStream = new BinaryInputStream(binaryStore, null); + if (inputStream.openFile(aFile) == false) { + return null; + }; + BinaryObjectData binaryObjectData; + BinaryHeader bh = inputStream.readHeader(); + if (bh==null) { + return null; + } + int inputFormat = bh.getHeaderFormat(); + while ((binaryObjectData = inputStream.readNextObject(inputFormat)) != null) { + if (binaryObjectData.getTimeMilliseconds() != 0) { + lastTime = binaryObjectData.getTimeMilliseconds(); + } + BinaryFooter bf; + switch (binaryObjectData.getObjectType()) { + case BinaryTypes.FILE_FOOTER: + // this is unlikely to happen, since we'll probably already have found an index file. + bf = new BinaryFooter(); + if (bf.readFooterData(binaryObjectData.getDataInputStream(), inputFormat)) { + if (bf.getDataDate() != 0) { + return bf; + } + } + break; + case BinaryTypes.MODULE_HEADER: + break; + case BinaryTypes.MODULE_FOOTER: + break; + case BinaryTypes.DATAGRAM: + break; + default: // should be data. + DataUnitBaseData baseData = binaryObjectData.getDataUnitBaseData(); + if (baseData != null) { + if (baseData.getTimeMilliseconds() != 0) { + lastTime = baseData.getTimeMilliseconds(); + } + if (baseData.getUID() != 0) { + lastUID = baseData.getUID(); + if (firstUID == null) { + firstUID = lastUID; + } + } + } + } + } + inputStream.closeFile(); + } + catch (Exception e) { + System.out.printf("Corrupt data file %s: %s\n", aFile, e.getMessage()); +// return null; + } + if (lastTime != null && lastUID != null) { + BinaryFooter bf = new BinaryFooter(); + bf.setHighestUID(lastUID); + bf.setLowestUID(firstUID); + bf.setDataDate(lastTime); + bf.setFileEndReason(BinaryFooter.END_CRASHED); + return bf; + } + else { + return null; + } + } + + /** + * Get the first header. This can be read from a data file whether or not there was a + * valid index file created. + * @param binFiles + * @return + */ + private BinaryHeader findFirstHeader(List binFiles) { + BinaryHeader binaryHead = new BinaryHeader(); + for (File aFile : binFiles) { + try { + DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(aFile))); + if (binaryHead.readHeader(dis)) { + return binaryHead; + } + } + catch (IOException e) { + continue; + } + } + return null; + } + + /** + * first simple status check to see if there are any files there at all. + */ + private int checkStoreStatus() { + String currDir = binaryStore.binaryStoreSettings.getStoreLocation(); + if (currDir == null) { + return StoreStatus.STATUS_MISSING; + } + File currfolder = new File(currDir); + if (currfolder.exists() == false) { + return StoreStatus.STATUS_MISSING; + } + // look for files in the folder. + boolean hasFiles = hasAnyFiles(currfolder); + if (hasFiles) { + return StoreStatus.STATUS_HASDATA; + } + else { + return StoreStatus.STATUS_EMPTY; + } + + } + + private boolean hasAnyFiles(File currFolder) { + PamFileFilter filefilter = new PamFileFilter("data files", ".pgdf"); + File[] list = currFolder.listFiles(filefilter); + if (list == null) { + return false; + } + for (int i = 0; i < list.length; i++) { + if (list[i].isDirectory()) { + if (hasAnyFiles(list[i])) { + return true; + } + } + if (list[i].getAbsolutePath().endsWith(".pgdf")) { + return true; + } + } + return false; + } + +} diff --git a/src/cpod/CPODControl.java b/src/cpod/CPODControl.java index 0df7731a..c095a825 100644 --- a/src/cpod/CPODControl.java +++ b/src/cpod/CPODControl.java @@ -19,6 +19,7 @@ import PamController.PamControlledUnitSettings; import PamController.PamController; import PamController.PamSettingManager; import PamController.PamSettings; +import PamController.fileprocessing.StoreStatus; import PamView.PamDetectionOverlayGraphics; import PamView.PamSymbol; import PamguardMVC.PamDataBlock; @@ -219,7 +220,7 @@ public class CPODControl extends OfflineFileControl implements PamSettings { int n = cpodLoader.loadData(dataBlock, usedMapPoints, offlineDataLoadInfo, loadObserver); return n >= 0; } - + } diff --git a/src/d3/D3Control.java b/src/d3/D3Control.java index 5a462f2c..61c3eff9 100644 --- a/src/d3/D3Control.java +++ b/src/d3/D3Control.java @@ -10,6 +10,7 @@ import java.util.List; import com.opencsv.CSVReader; import com.opencsv.exceptions.CsvException; +import PamController.fileprocessing.StoreStatus; import d3.calibration.CalFileReader; import d3.calibration.CalibrationInfo; import d3.calibration.CalibrationSet; @@ -487,4 +488,5 @@ public class D3Control extends OfflineFileControl { public D3DataPlotProvider getD3DataPlotProvider() { return d3DataPlotProvider; } + } diff --git a/src/decimator/DecimatorControl.java b/src/decimator/DecimatorControl.java index 2e4abf4f..7bf9f8ac 100644 --- a/src/decimator/DecimatorControl.java +++ b/src/decimator/DecimatorControl.java @@ -45,6 +45,7 @@ import PamController.PamController; import PamController.PamControllerInterface; import PamController.PamSettingManager; import PamController.PamSettings; +import PamController.fileprocessing.StoreStatus; import PamguardMVC.PamDataBlock; import PamguardMVC.PamDataUnit; import PamguardMVC.PamProcess; @@ -261,6 +262,6 @@ public class DecimatorControl extends PamControlledUnit implements PamSettings, double m = fbig % fsmall; return m == 0; } - + } diff --git a/src/difar/beamforming/BeamformControl.java b/src/difar/beamforming/BeamformControl.java index 3ce79544..edf3507b 100644 --- a/src/difar/beamforming/BeamformControl.java +++ b/src/difar/beamforming/BeamformControl.java @@ -17,6 +17,7 @@ import PamController.PamControlledUnitSettings; import PamController.PamController; import PamController.PamSettingManager; import PamController.PamSettings; +import PamController.fileprocessing.StoreStatus; import PamUtils.PamUtils; import PamguardMVC.PamDataBlock; import PamguardMVC.PamProcess; @@ -197,5 +198,6 @@ public class BeamformControl extends PamControlledUnit implements PamSettings, O // TODO Auto-generated method stub return beamformProcess; } + } \ No newline at end of file diff --git a/src/generalDatabase/DBControlUnit.java b/src/generalDatabase/DBControlUnit.java index 6baf2658..ff310652 100644 --- a/src/generalDatabase/DBControlUnit.java +++ b/src/generalDatabase/DBControlUnit.java @@ -18,12 +18,14 @@ import generalDatabase.backup.DatabaseBackupStream; import pamScrollSystem.ViewLoadObserver; import pamViewFX.pamTask.PamTaskUpdate; import PamController.AWTScheduler; +import PamController.DataOutputStore; import PamController.OfflineDataStore; import PamController.PamControlledUnit; import PamController.PamController; import PamController.PamControllerInterface; import PamController.PamGUIManager; import PamController.PamSettingManager; +import PamController.fileprocessing.StoreStatus; import PamController.status.ModuleStatus; import PamController.status.QuickRemedialAction; import PamguardMVC.PamDataBlock; @@ -39,7 +41,7 @@ import backupmanager.BackupInformation; * @see DBControl * */ -public class DBControlUnit extends DBControl implements OfflineDataStore { +public class DBControlUnit extends DBControl implements DataOutputStore { @@ -492,5 +494,15 @@ public class DBControlUnit extends DBControl implements OfflineDataStore { return backupInformation; } + @Override + public StoreStatus getStoreStatus(boolean getDetail) { + return getDbProcess().getStoreStatus(this, getDetail); + } + + @Override + public boolean deleteDataFrom(long timeMillis) { + return getDbProcess().deleteDataFrom(timeMillis); + } + } diff --git a/src/generalDatabase/DBProcess.java b/src/generalDatabase/DBProcess.java index 9e3bf699..167d49f5 100644 --- a/src/generalDatabase/DBProcess.java +++ b/src/generalDatabase/DBProcess.java @@ -1,6 +1,9 @@ package generalDatabase; import generalDatabase.ColumnMetaData.METACOLNAMES; +import generalDatabase.clauses.FixedClause; +import generalDatabase.clauses.FromClause; +import generalDatabase.clauses.PAMSelectClause; import generalDatabase.pamCursor.PamCursor; import generalDatabase.ucanAccess.UCanAccessSystem; @@ -14,6 +17,7 @@ import java.io.InputStreamReader; import java.io.PrintWriter; import java.sql.Connection; import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; @@ -21,6 +25,7 @@ import java.sql.SQLWarning; import java.sql.Statement; import java.sql.Types; import java.util.ArrayList; +import java.util.List; import javax.swing.JFileChooser; import javax.swing.JFrame; @@ -39,6 +44,7 @@ import loggerForms.formdesign.FormEditor; import PamController.PamControlledUnit; import PamController.PamController; import PamController.PamFolders; +import PamController.fileprocessing.StoreStatus; import PamUtils.PamCalendar; import PamUtils.PamFileChooser; import PamView.dialog.warn.WarnOnce; @@ -1435,4 +1441,141 @@ public class DBProcess extends PamProcess { } } + /** + * Get the store status for use with pre-process checks. + * @param getDetail get full details of start and end times. + * @return database store status. + */ + public StoreStatus getStoreStatus(DBControlUnit dbControlUnit, boolean getDetail) { + DatabaseStoreStatus dbStoreStatus = new DatabaseStoreStatus(dbControlUnit); + // and work out if any tables have anything in them already ... + int status = 0; + if (dbControlUnit.getConnection() == null) { + status = StoreStatus.STATUS_MISSING; + } + else { + boolean anyData = hasAnyOutputData(); + if (anyData) { + status = StoreStatus.STATUS_HASDATA; + } + else { + status = StoreStatus.STATUS_EMPTY; + } + } + if (status == StoreStatus.STATUS_HASDATA && getDetail) { + getStoreLimits(dbStoreStatus); + } + dbStoreStatus.setStoreStatus(status); + return dbStoreStatus; + } + + private void getStoreLimits(DatabaseStoreStatus dbStoreStatus) { + ArrayList allDataBlocks = PamController.getInstance().getDataBlocks(); + PamTableDefinition tableDefinition; + SQLLogging logging; + + // for each datablock, check that the process can log (ignoring GPS process) + for (int i = 0; i < allDataBlocks.size(); i++) { + PamDataBlock aBlock = allDataBlocks.get(i); + logging = aBlock.getLogging(); + if (logging == null) { + continue; + } + if (aBlock.getMixedDirection() != PamDataBlock.MIX_INTODATABASE) { + continue; // don't want things like GPS data. + } + getStoreLimits(aBlock, dbStoreStatus); + } + + } + + /** + * Get first and last records for a table. + * @param aBlock + * @param dbStoreStatus + */ + private void getStoreLimits(PamDataBlock aBlock, DatabaseStoreStatus dbStoreStatus) { + // TODO Auto-generated method stub + SQLLogging logging = aBlock.getLogging(); + PamConnection con = databaseControll.getConnection(); + SQLTypes sqlTypes = con.getSqlTypes(); + String q1 = String.format("SELECT MIN(UTC) FROM %s",sqlTypes.formatTableName(logging.getTableDefinition().getTableName())); + Long t = getUTC(con, q1); + dbStoreStatus.testFirstDataTime(t); + String q2 = String.format("SELECT MAX(UTC) FROM %s",sqlTypes.formatTableName(logging.getTableDefinition().getTableName())); + Long t2 = getUTC(con, q2); + dbStoreStatus.testLastDataTime(t2); + } + + private Long getUTC(PamConnection con, String qStr) { + + Object utcObject = null; + try { + PreparedStatement stmt = con.getConnection().prepareStatement(qStr); + ResultSet result = stmt.executeQuery(); + if (result.next()) { + utcObject = result.getObject(1); + + } + result.close(); + stmt.close(); + } catch (SQLException e) { + e.printStackTrace(); + return null; + } + if (utcObject == null) { + return null; + } + Long millis = SQLTypes.millisFromTimeStamp(utcObject); + return millis; + } + + /** + * Is there any data in any output tables ? + * @return + */ + private boolean hasAnyOutputData() { + ArrayList allDataBlocks = PamController.getInstance().getDataBlocks(); + PamTableDefinition tableDefinition; + SQLLogging logging; + + // for each datablock, check that the process can log (ignoring GPS process) + for (int i = 0; i < allDataBlocks.size(); i++) { + PamDataBlock aBlock = allDataBlocks.get(i); + logging = aBlock.getLogging(); + if (logging == null) { + continue; + } + if (aBlock.getMixedDirection() != PamDataBlock.MIX_INTODATABASE) { + continue; // don't want things like GPS data. + } + // get a record count. + Integer count = logging.countTableItems(null); + if (count != null && count > 0) { + return true; + } + } + + return false; + } + + public boolean deleteDataFrom(long timeMillis) { + ArrayList allDataBlocks = PamController.getInstance().getDataBlocks(); + PamTableDefinition tableDefinition; + SQLLogging logging; + + // for each datablock, check that the process can log (ignoring GPS process) + boolean ok = true; + for (int i = 0; i < allDataBlocks.size(); i++) { + PamDataBlock aBlock = allDataBlocks.get(i); + logging = aBlock.getLogging(); + if (logging == null) { + continue; + } + PAMSelectClause clause = new FromClause(timeMillis); + ok &= logging.deleteData(clause); + } + return ok; + } + } diff --git a/src/generalDatabase/DatabaseStoreStatus.java b/src/generalDatabase/DatabaseStoreStatus.java new file mode 100644 index 00000000..48243930 --- /dev/null +++ b/src/generalDatabase/DatabaseStoreStatus.java @@ -0,0 +1,43 @@ +package generalDatabase; + +import PamController.fileprocessing.StoreStatus; + +public class DatabaseStoreStatus extends StoreStatus { + + private DBControlUnit dbControl; + + public DatabaseStoreStatus(DBControlUnit dbControl) { + super(dbControl); + this.dbControl = dbControl; + } + + @Override + public long getFreeSpace() { + String name = dbControl.getDatabaseName(); // may not have the path, which is what we need. + return getFreeSpace(name); // this may not work, particularly for server based systems. + } + + public void testFirstDataTime(Long t) { + if (t == null) { + return; + } + if (getFirstDataTime() == null) { + setFirstDataTime(t); + } + if (t < getFirstDataTime()) { + setFirstDataTime(t); + } + } + public void testLastDataTime(Long t) { + if (t == null) { + return; + } + if (getLastDataTime() == null) { + setLastDataTime(t); + } + if (t > getLastDataTime()) { + setLastDataTime(t); + } + } + +} diff --git a/src/generalDatabase/SQLLogging.java b/src/generalDatabase/SQLLogging.java index 7cbd086d..8e4cc4e5 100644 --- a/src/generalDatabase/SQLLogging.java +++ b/src/generalDatabase/SQLLogging.java @@ -635,11 +635,20 @@ public abstract class SQLLogging { if (pamConn == null) { return null; } + SQLTypes sqlTypes = pamConn.getSqlTypes(); //the clause contains 'WHERE' so it's possible to make a null one. - String qStr = String.format("SELECT COUNT(%s.Id) FROM %s %s", - pamTableDefinition.getTableName(), - pamViewParameters.getSelectClause(sqlTypes)); + String qStr; + if (pamViewParameters == null) { + qStr = String.format("SELECT COUNT(Id) FROM %s", + pamTableDefinition.getTableName()); + } + else { + qStr = String.format("SELECT COUNT(%s.Id) FROM %s %s", + pamTableDefinition.getTableName(), + pamTableDefinition.getTableName(), + pamViewParameters.getSelectClause(sqlTypes)); + } int count = 0; try { PreparedStatement stmt = pamConn.getConnection().prepareStatement(qStr); diff --git a/src/pamguard/Pamguard.java b/src/pamguard/Pamguard.java index 4a142c0b..79069e9d 100644 --- a/src/pamguard/Pamguard.java +++ b/src/pamguard/Pamguard.java @@ -29,6 +29,7 @@ import PamController.PamGUIManager; import PamController.PamSettingManager; import PamController.PamguardVersionInfo; import PamController.pamBuoyGlobals; +import PamController.fileprocessing.ReprocessStoreChoice; import PamModel.SMRUEnable; import PamUtils.FileFunctions; import PamUtils.PamExceptionHandler; @@ -250,6 +251,15 @@ public class Pamguard { // auto exit at end of processing. GlobalArguments.setParam(PamController.AUTOEXIT, PamController.AUTOEXIT); } + else if (anArg.equalsIgnoreCase(ReprocessStoreChoice.paramName)) { + String arg = args[iArg++]; + ReprocessStoreChoice choice = ReprocessStoreChoice.valueOf(arg); + if (choice == null) { + String warn = String.format("Reprocessing storage input parameter %s value \"%s\" is not a recognised value", ReprocessStoreChoice.paramName, arg); + WarnOnce.showWarning("Invalid input parameter", warn, WarnOnce.WARNING_MESSAGE); + } + GlobalArguments.setParam(ReprocessStoreChoice.paramName, arg); + } else if (anArg.equalsIgnoreCase("-help")) { System.out.println("--PamGuard Help"); System.out.println("\n--For standard GUI deployment run without any options.\n"); From 5f9d29e1313d9d69ae39af6d246e413fc7b58db6 Mon Sep 17 00:00:00 2001 From: Douglas Gillespie <50671166+douggillespie@users.noreply.github.com> Date: Tue, 21 Feb 2023 16:01:50 +0000 Subject: [PATCH 4/5] Reprocessing Ironed out a few bugs in restarting processing. --- src/Acquisition/FileInputSystem.java | 7 ++++- src/Acquisition/FolderInputSystem.java | 3 ++ src/PamController/PamController.java | 13 +++++---- .../fileprocessing/ReprocessManager.java | 4 +++ src/RightWhaleEdgeDetector/RWEProcess.java | 1 + src/binaryFileStorage/BinaryOutputStream.java | 3 ++ src/binaryFileStorage/BinaryStore.java | 1 + src/binaryFileStorage/BinaryStoreDeleter.java | 16 +++++++---- .../BinaryStoreStatusFuncs.java | 28 ++++++++++++++----- 9 files changed, 58 insertions(+), 18 deletions(-) diff --git a/src/Acquisition/FileInputSystem.java b/src/Acquisition/FileInputSystem.java index 34f49ffa..2727ca05 100644 --- a/src/Acquisition/FileInputSystem.java +++ b/src/Acquisition/FileInputSystem.java @@ -558,7 +558,10 @@ public class FileInputSystem extends DaqSystem implements ActionListener, PamSe // if (fileInputParameters.recentFiles == null) return false; // if (fileInputParameters.recentFiles.size() < 1) return false; // String fileName = fileInputParameters.recentFiles.get(0); - return runFileAnalysis(); + if (runFileAnalysis() == false) { + return false; + } + return true; } public File getCurrentFile() { @@ -737,6 +740,8 @@ public class FileInputSystem extends DaqSystem implements ActionListener, PamSe @Override public boolean startSystem(AcquisitionControl daqControl) { + + if (audioStream == null) return false; dontStop = true; diff --git a/src/Acquisition/FolderInputSystem.java b/src/Acquisition/FolderInputSystem.java index 9ac7778c..19aff820 100644 --- a/src/Acquisition/FolderInputSystem.java +++ b/src/Acquisition/FolderInputSystem.java @@ -141,6 +141,7 @@ public class FolderInputSystem extends FileInputSystem implements PamSettings, D String[] selList = {globalFolder}; folderInputParameters.setSelectedFiles(selList); // need to immediately make the allfiles list since it's about to get used by the reprocess manager + // need to worry about how to wait for this since it's starting in a different thread. makeSelFileList(); } @@ -868,6 +869,8 @@ public class FolderInputSystem extends FileInputSystem implements PamSettings, D long fileStart = getFileStartTime(allFiles.get(i).getAbsoluteFile()); if (fileStart >= startTime) { currentFile = i; + PamCalendar.setSoundFile(true); + PamCalendar.setSessionStartTime(startTime); System.out.printf("Sound Acquisition start processing at file %s time %s\n", allFiles.get(i).getName(), PamCalendar.formatDBDateTime(fileStart)); return true; diff --git a/src/PamController/PamController.java b/src/PamController/PamController.java index 009af1bd..71f0f9ef 100644 --- a/src/PamController/PamController.java +++ b/src/PamController/PamController.java @@ -1212,6 +1212,8 @@ public class PamController implements PamControllerInterface, PamSettings { } /* + * + * This needs to be called after prepareproces. * Now we do some extra checks on the stores to see if we want to overwite data, * carry on from where we left off, etc. */ @@ -1221,14 +1223,15 @@ public class PamController implements PamControllerInterface, PamSettings { if (goonthen == false) { System.out.println("Data processing will not start since you've chosen not to overwrite existing output data"); pamStop(); + setPamStatus(PAM_IDLE); return false; } - - if (saveSettings) { - saveSettings(PamCalendar.getSessionStartTime()); - } } - + + if (saveSettings) { + saveSettings(PamCalendar.getSessionStartTime()); + } + StorageOptions.getInstance().setBlockOptions(); t1 = System.currentTimeMillis(); diff --git a/src/PamController/fileprocessing/ReprocessManager.java b/src/PamController/fileprocessing/ReprocessManager.java index 94323f67..69ebf54f 100644 --- a/src/PamController/fileprocessing/ReprocessManager.java +++ b/src/PamController/fileprocessing/ReprocessManager.java @@ -48,6 +48,10 @@ public class ReprocessManager { return true; } + if (choiceSummary.getInputStoreInfo() == null) { + return true; + } + // need to decide what to do based on the list of possible choices. ReprocessStoreChoice choice = chosePartStoreAction(choiceSummary); diff --git a/src/RightWhaleEdgeDetector/RWEProcess.java b/src/RightWhaleEdgeDetector/RWEProcess.java index 07a0584c..ecf4af68 100644 --- a/src/RightWhaleEdgeDetector/RWEProcess.java +++ b/src/RightWhaleEdgeDetector/RWEProcess.java @@ -177,6 +177,7 @@ public class RWEProcess extends PamProcess { minSoundType = rweControl.rweParameters.minSoundType; classifier.setSoundData(getSampleRate(), sourceDataBlock.getFftLength(), sourceDataBlock.getFftHop()); + System.out.println("Create right whale channel process " + iChannel); } diff --git a/src/binaryFileStorage/BinaryOutputStream.java b/src/binaryFileStorage/BinaryOutputStream.java index 6314c88e..1e474686 100644 --- a/src/binaryFileStorage/BinaryOutputStream.java +++ b/src/binaryFileStorage/BinaryOutputStream.java @@ -117,6 +117,8 @@ public class BinaryOutputStream { File outputFile = new File(mainFileName); boolean open = openPGDFFile(outputFile); + +// System.out.println("Open outout file " + outputFile.getAbsolutePath()); if (open) { addToDataMap(outputFile); @@ -260,6 +262,7 @@ public class BinaryOutputStream { public synchronized boolean closeFile() { boolean ok = true; +// System.out.println("Close output file " + mainFileName); if (dataOutputStream != null) { if (currentDataMapPoint != null) { currentDataMapPoint.setBinaryFooter(footer); diff --git a/src/binaryFileStorage/BinaryStore.java b/src/binaryFileStorage/BinaryStore.java index 3f3ebb02..f3965911 100644 --- a/src/binaryFileStorage/BinaryStore.java +++ b/src/binaryFileStorage/BinaryStore.java @@ -2404,6 +2404,7 @@ PamSettingsSource, DataOutputStore { } else { reportError("No valid header in file " + file.getAbsolutePath()); + inputStream.closeFile(); return null; } diff --git a/src/binaryFileStorage/BinaryStoreDeleter.java b/src/binaryFileStorage/BinaryStoreDeleter.java index 1497ec45..13401e04 100644 --- a/src/binaryFileStorage/BinaryStoreDeleter.java +++ b/src/binaryFileStorage/BinaryStoreDeleter.java @@ -142,9 +142,10 @@ public class BinaryStoreDeleter { boolean headOk = false; BinaryHeader binaryHead = new BinaryHeader(); try { - DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(indexFile))); + FileInputStream fis = new FileInputStream(indexFile); + DataInputStream dis = new DataInputStream(new BufferedInputStream(fis)); headOk = binaryHead.readHeader(dis); - dis.close(); + fis.close(); } catch (IOException e) { headOk = false; @@ -188,6 +189,7 @@ public class BinaryStoreDeleter { } private boolean partialCopyFile(PamDataBlock aBlock, File dataFile, long timeMillis) { + System.out.printf("Partial delete of file %s from %s\n", dataFile.getAbsoluteFile(), PamCalendar.formatDBDateTime(timeMillis)); try { BinaryInputStream inputStream = new BinaryInputStream(binaryStore, aBlock); if (inputStream.openFile(dataFile) == false) { @@ -303,10 +305,11 @@ public class BinaryStoreDeleter { * @return */ private boolean deleteFileSet(File dataFile) { + System.out.printf("Deleting full file set for %s\n", dataFile.getAbsoluteFile()); + boolean deleteOk = true; try { File indexFile = binaryStore.findIndexFile(dataFile, true); File noiseFile = binaryStore.findNoiseFile(dataFile, true); - boolean deleteOk = true; deleteOk &= dataFile.delete(); if (indexFile != null) { deleteOk &= indexFile.delete(); @@ -314,11 +317,14 @@ public class BinaryStoreDeleter { if (noiseFile != null) { deleteOk &= noiseFile.delete(); } - return deleteOk; } catch (Exception e) { - return false; + deleteOk = false; } + + System.out.printf("Deleting full file set %s for %s\n", deleteOk?"OK":"Error", dataFile.getAbsoluteFile()); + + return deleteOk; } diff --git a/src/binaryFileStorage/BinaryStoreStatusFuncs.java b/src/binaryFileStorage/BinaryStoreStatusFuncs.java index 31d9e9dc..0454655a 100644 --- a/src/binaryFileStorage/BinaryStoreStatusFuncs.java +++ b/src/binaryFileStorage/BinaryStoreStatusFuncs.java @@ -136,15 +136,16 @@ public class BinaryStoreStatusFuncs { * get the UID and time of the last item in the file. Can return these in the form of * a BinaryFooter since it's pretty much the same information needed. */ + BinaryInputStream inputStream = new BinaryInputStream(binaryStore, null); try { // need to work through the file now. - BinaryInputStream inputStream = new BinaryInputStream(binaryStore, null); if (inputStream.openFile(aFile) == false) { return null; }; BinaryObjectData binaryObjectData; BinaryHeader bh = inputStream.readHeader(); if (bh==null) { + inputStream.closeFile(); return null; } int inputFormat = bh.getHeaderFormat(); @@ -159,6 +160,7 @@ public class BinaryStoreStatusFuncs { bf = new BinaryFooter(); if (bf.readFooterData(binaryObjectData.getDataInputStream(), inputFormat)) { if (bf.getDataDate() != 0) { + inputStream.closeFile(); return bf; } } @@ -184,11 +186,18 @@ public class BinaryStoreStatusFuncs { } } } - inputStream.closeFile(); } catch (Exception e) { System.out.printf("Corrupt data file %s: %s\n", aFile, e.getMessage()); // return null; + } + try { + if (inputStream != null) { + inputStream.closeFile(); + } + } + catch (Exception e) { + } if (lastTime != null && lastUID != null) { BinaryFooter bf = new BinaryFooter(); @@ -211,18 +220,23 @@ public class BinaryStoreStatusFuncs { */ private BinaryHeader findFirstHeader(List binFiles) { BinaryHeader binaryHead = new BinaryHeader(); + DataInputStream dis = null; for (File aFile : binFiles) { try { - DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(aFile))); - if (binaryHead.readHeader(dis)) { - return binaryHead; - } + dis = new DataInputStream(new BufferedInputStream(new FileInputStream(aFile))); } catch (IOException e) { + binaryHead = null; continue; } + try { + dis.close(); + } + catch (IOException e) { + + } } - return null; + return binaryHead; } /** From 16e8184a2311bc14211a042b4b346957a728a434 Mon Sep 17 00:00:00 2001 From: Douglas Gillespie <50671166+douggillespie@users.noreply.github.com> Date: Wed, 1 Mar 2023 16:43:08 +0000 Subject: [PATCH 5/5] Batch processing Updates to support batch processing control --- src/PamModel/PamModel.java | 2 - src/PamView/component/DataBlockTableView.java | 59 +++++++++++++++---- src/generalDatabase/PamTableDefinition.java | 2 +- src/generalDatabase/PamTableItem.java | 5 +- src/generalDatabase/sqlite/SqliteSystem.java | 32 +++++++++- 5 files changed, 85 insertions(+), 15 deletions(-) diff --git a/src/PamModel/PamModel.java b/src/PamModel/PamModel.java index 38539cfe..4ce627dc 100644 --- a/src/PamModel/PamModel.java +++ b/src/PamModel/PamModel.java @@ -460,8 +460,6 @@ final public class PamModel implements PamModelInterface, PamSettings { mi.setModulesMenuGroup(utilitiesGroup); mi.setMaxNumber(1); - - /* * ************* End Utilities Group ******************* */ diff --git a/src/PamView/component/DataBlockTableView.java b/src/PamView/component/DataBlockTableView.java index 7000daa5..124addfc 100644 --- a/src/PamView/component/DataBlockTableView.java +++ b/src/PamView/component/DataBlockTableView.java @@ -5,6 +5,7 @@ import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.awt.event.MouseAdapter; import java.awt.event.MouseEvent; +import java.util.ArrayList; import javax.swing.JComponent; import javax.swing.JMenuItem; @@ -49,6 +50,14 @@ public abstract class DataBlockTableView { private SwingTableColumnWidths columnWidths; + /** + * Most work will run off a copy of the data. + * Makes it easier to include data selectors, etc. + */ + private ArrayList dataUnitCopy; + + private Object copySynch = new Object(); + public DataBlockTableView(PamDataBlock pamDataBlock, String displayName) { this.pamDataBlock = pamDataBlock; this.displayName = displayName; @@ -141,6 +150,9 @@ public abstract class DataBlockTableView { String tip = null; java.awt.Point p = e.getPoint(); int rowIndex = rowAtPoint(p); + if (rowIndex < 0) { + return null; + } int colIndex = columnAtPoint(p); int realColumnIndex = convertColumnIndexToModel(colIndex); T dataUnit = getDataUnit(rowIndex); @@ -201,10 +213,14 @@ public abstract class DataBlockTableView { * @return data unit for the table row. */ private final T getDataUnit(int tableRow) { - synchronized (pamDataBlock.getSynchLock()) { + synchronized (copySynch) { int rowIndex = getDataIndexForRow(tableRow); if (rowIndex < 0) return null; - return pamDataBlock.getDataUnit(rowIndex, PamDataBlock.REFERENCE_CURRENT); + if (dataUnitCopy == null) { + return null; + } + return dataUnitCopy.get(tableRow); +// return pamDataBlock.getDataUnit(rowIndex, PamDataBlock.REFERENCE_CURRENT); } } @@ -212,10 +228,13 @@ public abstract class DataBlockTableView { * Get the number of rows in the table - default behaviour is the * number of rows in the datablock, but this may be overridded if * data are being selected in a different way. - * @return numer of table rows to show. + * @return number of table rows to show. */ public int getRowCount() { - return pamDataBlock.getUnitsCount(); + if (dataUnitCopy == null) { + return 0; + } + return dataUnitCopy.size(); } /** @@ -227,7 +246,10 @@ public abstract class DataBlockTableView { * @return */ public int getDataIndexForRow(int tableRow) { - int nRow = pamDataBlock.getUnitsCount(); + if (dataUnitCopy == null) { + return tableRow; + } + int nRow = dataUnitCopy.size(); if (!isViewer) { tableRow = nRow-tableRow-1; } @@ -244,12 +266,14 @@ public abstract class DataBlockTableView { @Override public void addData(PamObservable o, PamDataUnit arg) { - blockTableModel.fireTableDataChanged(); + DataBlockTableView.this.updatePamData(); +// blockTableModel.fireTableDataChanged(); } @Override public void updateData(PamObservable observable, PamDataUnit pamDataUnit) { - blockTableModel.fireTableDataChanged(); + DataBlockTableView.this.updatePamData(); +// blockTableModel.fireTableDataChanged(); } @Override @@ -258,6 +282,15 @@ public abstract class DataBlockTableView { } } + + private void updatePamData() { + synchronized (copySynch) { + dataUnitCopy = pamDataBlock.getDataCopy(); + } + blockTableModel.fireTableDataChanged(); + } + + private class MouseAction extends MouseAdapter { /* (non-Javadoc) @@ -304,7 +337,11 @@ public abstract class DataBlockTableView { * @return Array of multiple rows selected. */ public T[] getMultipleSelectedRows() { - synchronized(pamDataBlock.getSynchLock()) { + if (dataUnitCopy == null) { + return null; + } +// synchronized(pamDataBlock.getSynchLock()) { // synch not needed with data copy. + synchronized (copySynch) { int[] selRows = testTable.getSelectedRows(); if (selRows == null) { return null; @@ -337,12 +374,14 @@ public abstract class DataBlockTableView { @Override public void scrollValueChanged(AbstractPamScroller abstractPamScroller) { - blockTableModel.fireTableDataChanged(); +// blockTableModel.fireTableDataChanged(); + updatePamData(); } @Override public void scrollRangeChanged(AbstractPamScroller pamScroller) { - blockTableModel.fireTableDataChanged(); +// blockTableModel.fireTableDataChanged(); + updatePamData(); } } diff --git a/src/generalDatabase/PamTableDefinition.java b/src/generalDatabase/PamTableDefinition.java index 6456671d..e8a2b5dc 100644 --- a/src/generalDatabase/PamTableDefinition.java +++ b/src/generalDatabase/PamTableDefinition.java @@ -13,7 +13,7 @@ import PamUtils.PamCalendar; * tables. Also used to prepare Sql statements for writing and * reading back data. * - * I did a bit of redifining what columns are used for on 4 Oct, 2012. + * I did a bit of redefining what columns are used for on 4 Oct, 2012. * PCLocalTime was a UTC time from the PC of the time analysis took place. * When running in real time, this would be the same as the data in the UTC column * (give or take the odd second for data to get through the system). I've now defined diff --git a/src/generalDatabase/PamTableItem.java b/src/generalDatabase/PamTableItem.java index dc523083..e329cc9e 100644 --- a/src/generalDatabase/PamTableItem.java +++ b/src/generalDatabase/PamTableItem.java @@ -225,7 +225,10 @@ public class PamTableItem implements Cloneable { // } public String getDeblankedStringValue() { - if (sqlType != Types.CHAR || value == null) { +// if (sqlType != Types.CHAR || value == null) { +// return null; +// } + if (value instanceof String == false) { return null; } return ((String) value).trim(); diff --git a/src/generalDatabase/sqlite/SqliteSystem.java b/src/generalDatabase/sqlite/SqliteSystem.java index b41d71b8..3ba3e90f 100644 --- a/src/generalDatabase/sqlite/SqliteSystem.java +++ b/src/generalDatabase/sqlite/SqliteSystem.java @@ -103,7 +103,10 @@ public class SqliteSystem extends DBSystem implements PamSettings { if (commandName == null) { return; } - setDatabaseName(commandName); + File commandFile = new File(commandName); + // check the file end is of the right type. Some batch systems may not get this right. + commandFile = PamFileFilter.checkFileEnd(commandFile, "sqlite3", true); + setDatabaseName(commandFile.getAbsolutePath()); } /** @@ -303,6 +306,30 @@ public class SqliteSystem extends DBSystem implements PamSettings { return true; } + @Override + public boolean checkDatabaseExists(String dbName) { + String commandName = GlobalArguments.getParam(DBControl.GlobalDatabaseNameArg); + if (commandName != null) { + return checkCommandLineDatabase(); + } + return super.checkDatabaseExists(dbName); + } + + private boolean checkCommandLineDatabase() { + String commandName = GlobalArguments.getParam(DBControl.GlobalDatabaseNameArg); + if (commandName == null) { + return false; + } + File dbFile = new File(commandName); + dbFile = PamFileFilter.checkFileEnd(dbFile, ".sqlite3", true); + commandName = dbFile.getAbsolutePath(); + if (dbFile.exists() == false) { + // create a new database without asking. + createNewDatabase(commandName); + } + return dbFile.exists(); + } + @Override public String getDatabaseName() { /* @@ -311,6 +338,9 @@ public class SqliteSystem extends DBSystem implements PamSettings { */ String commandName = GlobalArguments.getParam(DBControl.GlobalDatabaseNameArg); if (commandName != null) { + File dbFile = new File(commandName); + dbFile = PamFileFilter.checkFileEnd(dbFile, ".sqlite3", true); + commandName = dbFile.getAbsolutePath(); return commandName; }