Improve startup checks (#160)

* Ishmael Tethys output

Added necessaries for Tethys output from Ishmael detectors. Also found a pretty major bug in the spectrogram correlation detector, where for each block of data it was only testing the first sample of each block, not all samples, for being over threshold.

* Speed and algorithm improvements to Ish matched filter

Seems to be errors in correlation, didn't support multiple channels and
also used very old and slow FFT, so working to fix all three issues.

* Updated matched filter

Updated and working Matched filter, though still some thinking to do about how the scaling of this works, since currently scaled by the template, so whole thing is dependent on the input. Need to think of a better way to do this.

* Update match filt normalisation

Normalisation now correctly using both the template and the signal for normalisation so that it will be data amplitude independent.

* invFFT improvements

Use faster inverse FFT of real data in correlation / delay functions.

* Improve ifft's in other modules to improve TDOA speeds

* Sorting mess of spec plugin graphics

Have got the Ishmael ones scrolling, but when scrolling, there is an offset in the data due to the lag of the correlation functions. Quite hard to fix with available timing data

* Improve ish spectrogram plugin

Sorted scaling and scrollling problems.

* Improve startup checks

Rethread startup checks so that a progress bar shows when PAMGuard is checking input and output files at start up. Also include single file processing in checks.

src/Acquisition/FileInputSystem.java

@@ -13,9 +13,11 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Serializable;
 import java.text.DateFormat;
+import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.Calendar;
 import java.util.List;
+import java.util.TimeZone;
 
 import javax.sound.sampled.AudioFormat;
 import javax.sound.sampled.AudioFormat.Encoding;
@@ -48,6 +50,8 @@ import Acquisition.filedate.FileDateObserver;
 import Acquisition.filetypes.SoundFileType;
 import Acquisition.pamAudio.PamAudioFileFilter;
 import Acquisition.pamAudio.PamAudioFileManager;
+import PamController.DataInputStore;
+import PamController.InputStoreInfo;
 import PamController.PamControlledUnitSettings;
 import PamController.PamController;
 import PamController.PamSettingManager;
@@ -55,6 +59,7 @@ import PamController.PamSettings;
 import PamDetection.RawDataUnit;
 import PamUtils.PamCalendar;
 import PamUtils.PamFileChooser;
+import PamUtils.worker.filelist.WavFileType;
 import PamView.dialog.PamLabel;
 import PamView.dialog.warn.WarnOnce;
 import PamView.panel.PamPanel;
@@ -81,7 +86,7 @@ import wavFiles.ByteConverter;
  * @see FolderInputSystem
  *
  */
-public class FileInputSystem  extends DaqSystem implements ActionListener, PamSettings, FileDateObserver {
+public class FileInputSystem  extends DaqSystem implements ActionListener, PamSettings, FileDateObserver, DataInputStore {
 
 	public static final String sysType = "Audio File";
 
@@ -806,10 +811,10 @@ public class FileInputSystem  extends DaqSystem implements ActionListener, PamSe
 		if (getCurrentFile() == null) {
 			return;
 		}
-		double fileSecs = readFileSamples / getSampleRate();
+//		double fileSecs = readFileSamples / getSampleRate();
-		double analSecs = (stopTime - fileStartTime) / 1000.;
+//		double analSecs = (stopTime - fileStartTime) / 1000.;
-		System.out.println(String.format("File %s, SR=%dHz, length=%3.1fs took %3.1fs = %3.1fx real time",
+//		System.out.println(String.format("File %s, SR=%dHz, length=%3.1fs took %3.1fs = %3.1fx real time",
-				getCurrentFile().getName(), (int)getSampleRate(), fileSecs, analSecs, fileSecs / analSecs));
+//				getCurrentFile().getName(), (int)getSampleRate(), fileSecs, analSecs, fileSecs / analSecs));
 		fullyStopped = true;
 	}
 
@@ -915,7 +920,7 @@ public class FileInputSystem  extends DaqSystem implements ActionListener, PamSe
 				fileProgress.setValue(progress);
 				sayEta();
 				long now = System.currentTimeMillis();
-				if (lastProgressTime > 0 && totalSamples > lastProgressUpdate) {
+				if (lastProgressTime > 0 && totalSamples > lastProgressUpdate && now-lastProgressTime > 1000) {
 					double speed = (double) (totalSamples - lastProgressUpdate) /
 							getSampleRate() / ((now-lastProgressTime)/1000.);
 					speedLabel.setText(String.format(" (%3.1f X RT)", speed));
@@ -1035,11 +1040,11 @@ public class FileInputSystem  extends DaqSystem implements ActionListener, PamSe
 					long blockMillis = (int) ((newDataUnit.getStartSample() * 1000) / sampleRate);
 					//					newDataUnit.timeMilliseconds = blockMillis;
 					PamCalendar.setSoundFileTimeInMillis(blockMillis);
-					if (fileSamples > 0 && totalSamples - lastProgressUpdate >= getSampleRate()*2) {
+					long now = System.currentTimeMillis();
+					if (fileSamples > 0 && totalSamples - lastProgressUpdate >= getSampleRate()*2 && now-lastProgressTime>1000) {
 						int progress = (int) (1000 * readFileSamples / fileSamples);
 						fileProgress.setValue(progress);
 						sayEta();
-						long now = System.currentTimeMillis();
 						if (lastProgressTime > 0 && totalSamples > lastProgressUpdate) {
 							double speed = (double) (totalSamples - lastProgressUpdate) /
 									getSampleRate() / ((now-lastProgressTime)/1000.);
@@ -1187,6 +1192,9 @@ public class FileInputSystem  extends DaqSystem implements ActionListener, PamSe
 			fileProgress.setMinimum(0);
 			fileProgress.setMaximum(1000);
 			fileProgress.setValue(0);
+			etaLabel.setToolTipText("Estimated end time (Local time)");
+			speedLabel.setToolTipText("Process speed (factor above real time)");
+			fileProgress.setToolTipText("Progress through current file");
 		}
 		return statusPanel;
 	}
@@ -1212,14 +1220,20 @@ public class FileInputSystem  extends DaqSystem implements ActionListener, PamSe
 
 		long now = System.currentTimeMillis();
 		DateFormat df;
+		String str;
 		if (timeMs - now < (6 * 3600 * 1000)) {
-			df = DateFormat.getTimeInstance(DateFormat.MEDIUM);
+//			df = DateFormat.getTimeInstance(DateFormat.MEDIUM);
+			df = new SimpleDateFormat("HH:mm:ss");
+//			df = new SimpleDateFormat("yyyy-MM-dd HH:mm");
+//			str = PamCalendar.formatLocalDateTime(timeMs)
 		}
 		else {
-			df = DateFormat.getDateTimeInstance(DateFormat.LONG, DateFormat.MEDIUM);
+//			df = DateFormat.getDateTimeInstance(DateFormat.LONG, DateFormat.MEDIUM);
+			df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
 		}
 		Calendar c = Calendar.getInstance();
 		c.setTimeInMillis(timeMs);
+		TimeZone zone = c.getTimeZone();
 		etaLabel.setText("End " + df.format(c.getTime()));
 	}
 
@@ -1287,4 +1301,35 @@ public class FileInputSystem  extends DaqSystem implements ActionListener, PamSe
 			aType.selected(this);
 		}
 	}
+	@Override
+	public InputStoreInfo getStoreInfo(boolean detail) {
+//		System.out.println("FileInputSystem: Get store info start:");
+		File currentFile = getCurrentFile();
+		if (currentFile == null || currentFile.exists() == false) {
+			return null;
+		}
+		WavFileType wavType = new WavFileType(currentFile);
+		wavType.getAudioInfo();
+		long firstFileStart = getFileStartTime(currentFile.getAbsoluteFile());
+		float duration = wavType.getDurationInSeconds();
+		long fileEnd = (long) (firstFileStart + duration*1000.);
+		long[] allFileStarts = {firstFileStart};
+		long[] allFileEnds = {fileEnd};
+		InputStoreInfo storeInf = new InputStoreInfo(acquisitionControl, 1, firstFileStart, firstFileStart, fileEnd);
+		storeInf.setFileStartTimes(allFileStarts);
+		storeInf.setFileEndTimes(allFileEnds);
+		return storeInf;
+	}
+
+	@Override
+	public boolean setAnalysisStartTime(long startTime) {
+		// TODO Auto-generated method stub
+		return true;
+	}
+
+	@Override
+	public String getBatchStatus() {
+		// TODO Auto-generated method stub
+		return null;
+	}
 }

src/Acquisition/FolderInputSystem.java

@@ -674,7 +674,6 @@ public class FolderInputSystem extends FileInputSystem implements PamSettings, D
 		boolean ans = false;
 		if (!folderInputParameters.mergeFiles) return false;
 
-
 		long currFileStart = 0;
 		long currFileLength = 0;
 		long currFileEnd = 0;
@@ -701,6 +700,9 @@ public class FolderInputSystem extends FileInputSystem implements PamSettings, D
 			currFileEnd += lastBit;
 		}
 		if (++currentFile < allFiles.size()) {
+			
+			calculateETA();
+			
 			long newStartTime = getFileStartTime(getCurrentFile());
 			long diff = newStartTime - currFileEnd;
 			if (diff > 2000 || diff < -5000 || newStartTime == 0) {
@@ -759,11 +761,12 @@ public class FolderInputSystem extends FileInputSystem implements PamSettings, D
 		if (currentFile > 0 && currentFile >= allFiles.size()) {
 			fileListComplete();
 		}
-		System.out.println("FolderinputSytem: daqHasEnded");
+//		System.out.println("FolderinputSytem: daqHasEnded");
 	}
 
 	private void setFolderProgress() {
 		folderProgress.setValue(currentFile);
+		folderProgress.setString(String.format("%d/%d", currentFile, folderProgress.getMaximum()));
 	}
 
 	protected void calculateETA() {
@@ -789,6 +792,7 @@ public class FolderInputSystem extends FileInputSystem implements PamSettings, D
 			barBit.add(folderProgress);
 			barBit.add(new PamLabel("   "));
 			barBit.add(super.getStatusBarComponent());
+			folderProgress.setToolTipText("Process through file folder(s)");
 		}
 		return barBit;
 	}

src/PamController/PamController.java

@@ -42,6 +42,7 @@ import PamController.command.NetworkController;
 import PamController.command.TerminalController;
 import PamController.command.WatchdogComms;
 import PamController.fileprocessing.ReprocessManager;
+import PamController.fileprocessing.ReprocessManagerMonitor;
 import PamController.masterReference.MasterReferencePoint;
 import PamController.settings.BatchViewSettingsImport;
 import PamController.settings.output.xml.XMLWriterDialog;
@@ -1287,17 +1288,68 @@ public class PamController implements PamControllerInterface, PamSettings {
 		 * off, etc.
 		 */
 		if (saveSettings && getRunMode() == RUN_NORMAL) { // only true on a button press or network start.
-			ReprocessManager reprocessManager = new ReprocessManager();
+			checkReprocessManager(saveSettings, startTime);
-			boolean goonthen = reprocessManager.checkOutputDataStatus();
-			if (!goonthen) {
-				System.out.println(
-						"Data processing will not start since you've chosen not to overwrite existing output data");
-				pamStop();
-				setPamStatus(PAM_IDLE);
-				return false;
-			}
 		}
+		else {
+			return continueStart(saveSettings, startTime);
+		}
+		return true;
+	}
 	
+	/**
+	 * Check the reprocess manager in a swing worker thread. 
+	 * @param saveSettings
+	 * @param startTime
+	 */
+	private void checkReprocessManager(boolean saveSettings, long startTime) {
+		ReprocessMon mon = new ReprocessMon(saveSettings, startTime);
+		ReprocessManager reprocessManager = new ReprocessManager();
+//		boolean goonthen = reprocessManager.checkOutputDataStatus();
+//		if (!goonthen) {
+//			System.out.println(
+//					"Data processing will not start since you've chosen not to overwrite existing output data");
+//			pamStop();
+//			setPamStatus(PAM_IDLE);
+//			return false;
+//		}
+		reprocessManager.startCheckingThread(getMainFrame(), mon);
+	}
+	
+	private class ReprocessMon implements ReprocessManagerMonitor {
+		private boolean saveSettings;
+		private long startTime;
+		public ReprocessMon(boolean saveSettings, long startTime) {
+			super();
+			this.saveSettings = saveSettings;
+			this.startTime = startTime;
+		}
+		@Override
+		public void done(boolean continueStart) {
+			reprocessManagerDone(continueStart, continueStart, startTime);
+		}
+	}
+	
+	private void reprocessManagerDone(boolean goonthen, boolean saveSettings, long startTime) {
+		if (!goonthen) {
+			System.out.println(
+					"Data processing will not start since you've chosen not to overwrite existing output data");
+			pamStop();
+			setPamStatus(PAM_IDLE);
+		}
+		else {
+			continueStart(saveSettings, startTime);
+		}
+	}
+
+	/**
+	 * Second half of the start process. This was originally in pamStart, but had
+	 * to be split out, so that the reprocessManager checks can run in a separate
+	 * thread in order to display a progress bar as files are catalogued. 
+	 * @param saveSettings
+	 * @param startTime
+	 * @return
+	 */
+	public boolean continueStart(boolean saveSettings, long startTime) {
 		if (saveSettings) {
 			startTime = PamCalendar.getSessionStartTime();
 //			System.out.printf("Saving settings for start time %s\n", PamCalendar.formatDBDateTime(startTime));
@@ -1313,7 +1365,8 @@ public class PamController implements PamControllerInterface, PamSettings {
 
 		StorageOptions.getInstance().setBlockOptions();
 
-		t1 = System.currentTimeMillis();
+		long t1 = System.currentTimeMillis();
+		ArrayList<PamControlledUnit> pamControlledUnits = pamConfiguration.getPamControlledUnits();
 		for (int iU = 0; iU < pamControlledUnits.size(); iU++) {
 			pamControlledUnits.get(iU).pamToStart();
 			// long t2 = System.currentTimeMillis();
@@ -1486,7 +1539,7 @@ public class PamController implements PamControllerInterface, PamSettings {
 		 * If it's running in multithreading mode, then at this point it is necessary to
 		 * make sure that all internal datablock buffers have had time to empty.
 		 */
-		System.out.println("Arrived in PamStopped() in thread " + Thread.currentThread().toString());
+//		System.out.println("Arrived in PamStopped() in thread " + Thread.currentThread().toString());
 
 		ArrayList<PamControlledUnit> pamControlledUnits = pamConfiguration.getPamControlledUnits();
 

src/PamController/fileprocessing/ReprocessManager.java

@@ -1,6 +1,10 @@
 package PamController.fileprocessing;
 
+import java.awt.Frame;
 import java.util.ArrayList;
+import java.util.List;
+
+import javax.swing.SwingWorker;
 
 import PamController.DataInputStore;
 import PamController.DataOutputStore;
@@ -11,6 +15,9 @@ import PamController.PamController;
 import PamController.PamGUIManager;
 import PamController.RawInputControlledUnit;
 import PamUtils.PamCalendar;
+import PamUtils.worker.PamWorkDialog;
+import PamUtils.worker.PamWorkMonitor;
+import PamUtils.worker.PamWorkProgressMessage;
 import PamView.dialog.warn.WarnOnce;
 import pamguard.GlobalArguments;
 
@@ -22,6 +29,98 @@ import pamguard.GlobalArguments;
  */
 public class ReprocessManager {
 
+	private volatile PamWorkDialog workDialog;
+	
+	private Object synch = new Object();
+	/**
+	 * Start a Swing worker thread to do the checks and to display a progress bar while doing it.<p>
+	 * Then when it's done, send the result to the monitor, which is basically telling PamController
+	 * whether or not to continue with start up 
+	 * @param mainFrame
+	 * @param mon monitor for final status message / instruction. 
+	 */
+	public void startCheckingThread(Frame mainFrame, ReprocessManagerMonitor mon) {
+		CheckWorker checkWorker = new CheckWorker(mainFrame, mon);
+		checkWorker.execute();	
+		synchronized (synch) {
+			workDialog = new PamWorkDialog(mainFrame, 1, "Checking input files and existing output data");
+			workDialog.setVisible(true);
+		}
+	}
+	
+	private void closeWorkDialog() {
+		/**
+		 * This will only get called when job has finished - but that might happen before the
+		 * dialog is even open, so wait for up to a second for it to appear before closing it anyway. 
+		 */
+		long t = System.currentTimeMillis();
+		while (System.currentTimeMillis()-t < 1000) {
+			if (workDialog != null) {
+				break;
+			}
+			try {
+				Thread.sleep(100);
+			} catch (InterruptedException e) {
+			}
+		}
+		synchronized (synch) {
+			if (workDialog != null) {
+				workDialog.setVisible(false);
+				workDialog.dispose();
+				workDialog = null;
+			}
+		}
+
+	}
+	
+	private class CheckWorker extends SwingWorker<Boolean, PamWorkProgressMessage> implements PamWorkMonitor {
+
+		private Frame mainFram;
+		private ReprocessManagerMonitor mon;
+		private volatile boolean result;
+		
+		public CheckWorker(Frame mainFram, ReprocessManagerMonitor mon) {
+			super();
+			this.mainFram = mainFram;
+			this.mon = mon;
+		}
+
+		@Override
+		protected Boolean doInBackground() throws Exception {
+			try {
+				result = checkOutputDataStatus(this);
+			}
+			catch (Exception e) {
+				e.printStackTrace();
+			}
+			return result;
+		}
+
+		@Override
+		protected void process(List<PamWorkProgressMessage> chunks) {
+			for (PamWorkProgressMessage message : chunks) {
+				synchronized(synch) {
+					if (workDialog != null) {
+						workDialog.update(message);
+					}
+				}
+			}
+		}
+
+		@Override
+		protected void done() {
+			closeWorkDialog();
+			mon.done(result);
+		}
+
+		@Override
+		public void update(PamWorkProgressMessage message) {
+			this.publish(message);
+		}
+		
+	}
+
+
 	/**
 	public ReprocessManager() {
 		// TODO Auto-generated constructor stub
@@ -32,10 +131,13 @@ public class ReprocessManager {
 	 *  we may not want to start again.  
 	 */
 	public boolean checkOutputDataStatus() {
+		return checkOutputDataStatus(null);
+	}
+	public boolean checkOutputDataStatus(PamWorkMonitor workMonitor) {
 		
 		StoreChoiceSummary choiceSummary = null;
 		if (isOfflineFiles()) {
-			choiceSummary = checkIOFilesStatus();
+			choiceSummary = checkIOFilesStatus(workMonitor);
 		}
 		else {
 			/*
@@ -149,9 +251,10 @@ public class ReprocessManager {
 	/**
 	 * Check the output of current files and databases and return a flag to PamController saying whether or
 	 * not processing should actually start, possibly overwriting, or if we need to not start to avoid overwriting. 
+	 * @param workMonitor 
 	 * @return true if processing should start. 
 	 */
-	private StoreChoiceSummary checkIOFilesStatus() {	
+	private StoreChoiceSummary checkIOFilesStatus(PamWorkMonitor workMonitor) {	
 		/**
 		 * Get information about the input. 
 		 * 
@@ -161,8 +264,12 @@ public class ReprocessManager {
 			return new StoreChoiceSummary(null, ReprocessStoreChoice.STARTNORMAL);
 		}
 		InputStoreInfo inputInfo = null;
+		
 		for (PamControlledUnit aPCU : inputStores) {
 			DataInputStore inputStore = (DataInputStore) aPCU;
+			if (workMonitor != null) {
+				workMonitor.update(new PamWorkProgressMessage(-1, "Checking input data " + aPCU.getUnitName()));
+			}
 			inputInfo = inputStore.getStoreInfo(true);
 //			System.out.println("Input store info: " + inputInfo);
 		}
@@ -179,6 +286,9 @@ public class ReprocessManager {
 		boolean partStores = false; 
 		int nOutputStores = 0;
 		for (PamControlledUnit aPCU : outputStores) {
+			if (workMonitor != null) {
+				workMonitor.update(new PamWorkProgressMessage(-1, "Checking output data " + aPCU.getUnitName()));
+			}
 			DataOutputStore offlineStore = (DataOutputStore) aPCU;
 			StoreStatus status = offlineStore.getStoreStatus(false);
 			nOutputStores++;

src/PamController/fileprocessing/ReprocessManagerMonitor.java

@@ -0,0 +1,7 @@
+package PamController.fileprocessing;
+
+public interface ReprocessManagerMonitor {
+
+	public void done(boolean continueStart);
+	
+}

src/PamUtils/worker/PamWorkMonitor.java

@@ -0,0 +1,7 @@
+package PamUtils.worker;
+
+public interface PamWorkMonitor {
+
+	public void update(PamWorkProgressMessage message);
+
+}

src/PamView/panel/PamProgressBar.java

@@ -1,7 +1,5 @@
 package PamView.panel;
 
-import java.awt.Dimension;
-
 import javax.swing.BoundedRangeModel;
 import javax.swing.JProgressBar;
 

src/binaryFileStorage/BinaryOutputStream.java

@@ -601,8 +601,7 @@ public class BinaryOutputStream {
 			opStream = new DataOutputStream(new BufferedOutputStream(new 
 					FileOutputStream(indexFile)));
 		} catch (FileNotFoundException e) {
-			// TODO Auto-generated catch block
+			System.err.println("Error creating index file: " + e.getMessage());
-			e.printStackTrace();
 			return false;
 		}