Merge branch 'restart' into main

dependency-reduced-pom.xml

@@ -4,7 +4,7 @@
   <groupId>org.pamguard</groupId>
   <artifactId>Pamguard</artifactId>
   <name>Pamguard Java12+</name>
-  <version>2.02.10b</version>
+  <version>2.02.10aa</version>
   <description>Pamguard for Java 12+, using Maven to control dependcies</description>
   <url>www.pamguard.org</url>
   <organization>

src/Acquisition/AcquisitionProcess.java

@@ -211,10 +211,10 @@ public class AcquisitionProcess extends PamProcess implements DataInputStore {
 
 		if (systemPrepared == false) return;
 
+		newDataQueue.clearList(); // clear this first to make sure nothing new comes in. 
+
 		//		before starting, clear all old data
 		rawDataBlock.clearAll();
-
-		newDataQueue.clearList();
 		
 		sampleRateErrorFilter.prepareFilter();
 		totalExtraSamples = 0;
@@ -302,6 +302,7 @@ public class AcquisitionProcess extends PamProcess implements DataInputStore {
 		// called by PamController.
 		// stop the running system - not the selected system since
 		// this may have changed
+				
 		restartTimer.stop();
 //		stallCheckTimer.stop();
 		pamStop("");
@@ -471,6 +472,9 @@ public class AcquisitionProcess extends PamProcess implements DataInputStore {
 	 * have been emptied and processing has stopped
 	 */
 	protected void pamHasStopped() {
+		
+		newDataQueue.clearList(); // clear this first to make sure nothing new comes in. 
+		
 		if (runningSystem == null){
 			runningSystem = acquisitionControl.findDaqSystem(null);
 		}
@@ -650,9 +654,10 @@ public class AcquisitionProcess extends PamProcess implements DataInputStore {
 
 			restartTimer.stop();
 
-			PamController.getInstance().pamStop();
-
-			PamController.getInstance().pamStart(false);
+			PamController.getInstance().restartPamguard();
+//			PamController.getInstance().pamStop();
+//
+//			PamController.getInstance().pamStart(false);
 
 		}
 

src/Map/MapRectProjector.java

@@ -404,7 +404,7 @@ public class MapRectProjector extends MapProjector {
 
 	private String findGpsTrackText(Point mousePoint, int ploNumberMatch) {
 		GPSControl gpsControl = GPSControl.getGpsControl();
-		if (gpsControl == null) {
+		if (gpsControl == null || mousePoint == null) {
 			return null;
 		}
 		LatLong currentPos = getDataPosition(new Coordinate3d(mousePoint.x, mousePoint.y));

src/PamController/PamController.java

@@ -128,6 +128,10 @@ public class PamController implements PamControllerInterface, PamSettings {
 	public static final int PAM_COMPLETE = 6;
 	public static final int PAM_MAPMAKING = 7;
 	public static final int PAM_OFFLINETASK = 8;
+	
+	public static final int BUTTON_START = 1;
+	public static final int BUTTON_STOP = 2;
+	private volatile int lastStartStopButton = 0;
 
 	// status' for RunMode = RUN_PAMVIEW
 	public static final int PAM_LOADINGDATA = 2;
@@ -161,7 +165,7 @@ public class PamController implements PamControllerInterface, PamSettings {
 	/**
 	 * The current PAM status
 	 */
-	private transient int pamStatus = PAM_IDLE;
+	private volatile int pamStatus = PAM_IDLE;
 
 	/**
 	 * PamGuard view params. 
@@ -240,6 +244,9 @@ public class PamController implements PamControllerInterface, PamSettings {
 	private Thread statusCheckThread;
 	private WaitDetectorThread detectorEndThread;
 	private boolean firstDataLoadComplete;
+	// keep a track of the total number of times PAMGuard is started for debug purposes. 
+	private int nStarts;
+	private RestartRunnable restartRunnable;
 
 
 	private PamController(int runMode, Object object) {
@@ -1045,8 +1052,45 @@ public class PamController implements PamControllerInterface, PamSettings {
 	 */
 	public void restartPamguard() {
 		pamStop();
-		startLater();		
+		
+		/*
+		 *  launch a restart thread, that won't do ANYTHING until
+		 *  PAMGuard is really idle and buffers are cleared. Can only
+		 *  have one of these at a time ! 
+		 */
+		if (restartRunnable != null) {
+			System.out.println("Warning !!!! PAMGuard is already trying to restart!");
+			return;
+		}
+		restartRunnable = new RestartRunnable();
+		Thread restartThread = new Thread(restartRunnable, "RestartPAMGuard Thread");
+		restartThread.run();
 	}
+	
+	private class RestartRunnable implements Runnable {
+
+		@Override
+		public void run() {
+			long t1 = System.currentTimeMillis();
+			while (getPamStatus() != PAM_IDLE) {
+				try {
+					Thread.sleep(200);
+				} catch (InterruptedException e) {
+
+				}
+			}
+			long t2 = System.currentTimeMillis();
+			restartRunnable = null;
+			System.out.printf("PAMGuard safe to restart after %d milliseconds\n", t2-t1);
+			startLater(false);	
+			
+		}
+		
+	}
+	
+	
+	
+	
 	/**
 	 * calls pamStart using the SwingUtilities
 	 * invokeLater command to start PAMGAURD 
@@ -1078,7 +1122,13 @@ public class PamController implements PamControllerInterface, PamSettings {
 
 		@Override
 		public void run() {
-			pamStart(saveSettings);
+			/*
+			 *  do a final check that the stop button hasn't been pressed - can arrive a bit
+			 *  late if the system was continually restarting.  
+			 */
+			if (lastStartStopButton != BUTTON_STOP) {
+				pamStart(saveSettings);
+			}
 		}
 	}
 
@@ -1103,6 +1153,26 @@ public class PamController implements PamControllerInterface, PamSettings {
 		}
 	}
 
+	/**
+	 * Called from the start button. A little book keeping 
+	 * to distinguish this from automatic starts / restarts
+	 * @return true if started.
+	 */
+	@Override
+	public boolean manualStart() {
+		lastStartStopButton = BUTTON_START;
+		return pamStart();
+	}
+	
+	/**
+	 * Called from the stop button. A little book keeping 
+	 * to distinguish this from automatic starts / restarts
+	 */
+	@Override
+	public void manualStop() {
+		lastStartStopButton = BUTTON_STOP;
+		pamStop();
+	}
 
 	/**
 	 * Start PAMGUARD. This function also gets called from the 
@@ -1193,6 +1263,12 @@ public class PamController implements PamControllerInterface, PamSettings {
 			saveSettings(PamCalendar.getSessionStartTime());
 		}
 
+		if (++nStarts > 1) {
+			// do this here - all processses should have reset buffers to start again by now. 
+			String msg = String.format("Starting PAMGuard go %d", nStarts);
+			dumpBufferStatus(msg, false);
+		}
+
 		StorageOptions.getInstance().setBlockOptions();
 
 		t1 = System.currentTimeMillis();
@@ -1254,6 +1330,7 @@ public class PamController implements PamControllerInterface, PamSettings {
 			}
 		}
 		
+		dumpBufferStatus("In stopping", false);
 		/*
 		 *  now launch another thread to wait for everything to have stopped, but
 		 *  leave this function so that AWT is released and graphics can update, the
@@ -1281,9 +1358,11 @@ public class PamController implements PamControllerInterface, PamSettings {
 				long t2 = System.currentTimeMillis();
 				if (t2 - t1 > 5000) {
 					System.out.printf("Stopping, but stuck in loop for CheckRunStatus for %3.1fs\n", (double) (t2-t1)/1000.);
+					dumpBufferStatus("Stopping stuck in loop", false);
+					break; // crap out anyway.
 				}
 				try {
-					Thread.sleep(10);
+					Thread.sleep(100);
 				} catch (InterruptedException e) {
 					e.printStackTrace();
 				}
@@ -1294,19 +1373,42 @@ public class PamController implements PamControllerInterface, PamSettings {
 		
 	}
 	
+	/**
+	 * Look in every data block, particularly threaded ones, and dump
+	 * the buffer status. This will have to go via PamProcess so that 
+	 * additional information can be added from any processes that 
+	 * hold additional data in other internal buffers. 
+	 * @param message Message to print prior to dumping buffers for debug. 
+	 * @param sayEmpties dump info even if a buffer is empty (otherwise, only ones that have stuff still)
+	 */
+	public void dumpBufferStatus(String message, boolean sayEmpties) {
+		//if (2 >1) return;
+		System.out.println("**** Dumping process buffer status: " + message);
+		ArrayList<PamControlledUnit> pamControlledUnits = pamConfiguration.getPamControlledUnits();
+		for (PamControlledUnit aUnit : pamControlledUnits) {
+			int numProcesses = aUnit.getNumPamProcesses();
+			for (int i=0; i<numProcesses; i++) {
+				PamProcess aProcess = aUnit.getPamProcess(i);
+				aProcess.dumpBufferStatus(message, sayEmpties);
+			}
+		}
+		System.out.println("**** End of process buffer dump: " + message);
+	}
+	
 	/**
 	 * Called once the detectors have actually stopped and puts a few finalising 
 	 * functions into the AWT thread. 
 	 */
 	private void finishStopping() {
 		detectorEndThread = null;
-		SwingUtilities.invokeLater(new Runnable() {
-			
-			@Override
-			public void run() {
+		// this was never getting invoked for some reason. 
+//		SwingUtilities.invokeLater(new Runnable() {
+//			
+//			@Override
+//			public void run() {
 				pamStopped();
-			}
-		});
+//			}
+//		});
 	}
 		
 	
@@ -1320,6 +1422,8 @@ public class PamController implements PamControllerInterface, PamSettings {
 		 * it is necessary to make sure that all internal datablock 
 		 * buffers have had time to empty.
 		 */
+		System.out.println("Arrived in PamStopped() in thread " + Thread.currentThread().toString());
+		
 		ArrayList<PamControlledUnit> pamControlledUnits = pamConfiguration.getPamControlledUnits();
 		
 		if (PamModel.getPamModel().isMultiThread()) {
@@ -1327,7 +1431,7 @@ public class PamController implements PamControllerInterface, PamSettings {
 				pamControlledUnits.get(iU).flushDataBlockBuffers(2000);
 			}
 		}
-		setPamStatus(PAM_IDLE);
+		dumpBufferStatus("In pamStopped, now idle", true);
 
 		// wait here until the status has changed to Pam_Idle, so that we know
 		// that we've really finished processing all data
@@ -1350,6 +1454,8 @@ public class PamController implements PamControllerInterface, PamSettings {
 		
 		long stopTime = PamCalendar.getTimeInMillis();
 		saveEndSettings(stopTime);
+
+		setPamStatus(PAM_IDLE);
 		
 		// no good having this here since it get's called at the end of every file. 
 //		if (GlobalArguments.getParam(PamController.AUTOEXIT) != null) {
@@ -1960,7 +2066,8 @@ public class PamController implements PamControllerInterface, PamSettings {
 		/*
 		 * This only get's called once when set idle at viewer mode startup. 
 		 */
-//		System.out.printf("*******   PamController.setPamStatus to %d, real status is %d\n",  pamStatus, getRealStatus());
+		System.out.printf("*******   PamController.setPamStatus to %d, real status is %d set in thread %s\n",  
+				pamStatus, getRealStatus(), Thread.currentThread().toString());
 		if (getRunMode() != RUN_PAMVIEW) {
 			TopToolBar.enableStartButton(pamStatus == PAM_IDLE);
 			TopToolBar.enableStopButton(pamStatus == PAM_RUNNING);
@@ -2054,6 +2161,7 @@ public class PamController implements PamControllerInterface, PamSettings {
 			statusWarning.setWarningMessage(warningMessage);
 			statusWarning.setWarnignLevel(1);
 			warningSystem.addWarning(statusWarning);
+//			System.out.println(warningMessage);
 		}
 	}
 

src/PamController/PamControllerInterface.java

@@ -458,6 +458,17 @@ public interface PamControllerInterface {
 	 * Close all modules and free up resources. 
 	 */
 	public void pamClose();
+
+	/**
+	 * Start function called from button to do a bit of extra book keeping
+	 * @return
+	 */
+	public boolean manualStart();
+
+	/**
+	 * Stop function called from button to do a bit of extra book keeping
+	 */
+	public void manualStop();
 	
 	//public void controllerAddFileMenuItem();
 

src/PamController/PamguardVersionInfo.java

@@ -16,7 +16,7 @@ public class PamguardVersionInfo {
 	 * @return release type 
 	 */
 	static public ReleaseType getReleaseType() {
-		return ReleaseType.CORE; 
+		return ReleaseType.OTHER; 
 	}
 	
 	/**

src/PamView/PamGui.java

@@ -1268,7 +1268,7 @@ public class PamGui extends PamView implements WindowListener, PamSettings {
 
 	class menuPamStart implements ActionListener {
 		public void actionPerformed(ActionEvent ev){
-			pamControllerInterface.pamStart();
+			pamControllerInterface.manualStart();
 
 		}
 	}
@@ -1285,7 +1285,7 @@ public class PamGui extends PamView implements WindowListener, PamSettings {
 
 	class menuPamStop implements ActionListener {
 		public void actionPerformed(ActionEvent ev){
-			pamControllerInterface.pamStop();
+			pamControllerInterface.manualStop();
 			//			enableLoggingMenu();		
 		}
 	}

src/PamguardMVC/PamDataBlock.java

@@ -2164,7 +2164,7 @@ public class PamDataBlock<Tunit extends PamDataUnit> extends PamObservable {
 	 */
 	@Override
 	public String toString() {
-		return getDataName();
+		return getLongDataName();
 	}
 
 	/**
@@ -4286,4 +4286,23 @@ public class PamDataBlock<Tunit extends PamDataUnit> extends PamObservable {
 		inputEl.setAttribute("Channels", String.format("0x%X", getChannelMap()));
 		return inputEl;
 	}
+
+	/**
+	 * Look in every data block, particularly threaded ones, and dump
+	 * the buffer status. This will have to go via PamProcess so that 
+	 * additional information can be added from any processes that 
+	 * hold additional data in other internal buffers. 
+	 * @param message Message to print prior to dumping buffers for debug. 
+	 * @param sayEmpties dump info even if a buffer is empty (otherwise, only ones that have stuff still)
+	 */
+	public void dumpBufferStatus(String message, boolean sayEmpties) {
+		int nObs = countObservers();
+		for (int i = 0; i < nObs; i++) {
+			PamObserver obs = getPamObserver(i);
+			if (obs instanceof ThreadedObserver) {
+				ThreadedObserver tObs = (ThreadedObserver) obs;
+				tObs.dumpBufferStatus(message, sayEmpties);
+			}
+		}
+	}
 }

src/PamguardMVC/PamObservable.java

@@ -208,6 +208,14 @@ public class PamObservable {//extends PanelOverlayDraw {
 			if (System.currentTimeMillis() - startTime > timeOutms) {
 				// have taken too long, so return that we've failed. 
 				System.out.println("Wait timeout in threaded observer");
+				// and clear everything that's left. 
+				for (int i = 0; i < pamObservers.size(); i++) {
+					pamObserver = pamObservers.get(i);
+					if (pamObserver.getClass() == ThreadedObserver.class) {
+						threadedObserver = (ThreadedObserver) pamObserver;
+						threadedObserver.clearEverything();
+					}
+				}
 				return false;
 			}
 			try {

src/PamguardMVC/PamProcess.java

@@ -767,7 +767,7 @@ abstract public class PamProcess implements PamObserver, ProcessAnnotator {
 		}
 	});
 
-	private int lastSourceNotificationType;
+	private volatile int lastSourceNotificationType;
 
 	private Object lastSourceNotificationObject;
 	
@@ -1065,4 +1065,24 @@ abstract public class PamProcess implements PamObserver, ProcessAnnotator {
 		return lastSourceNotificationObject;
 	}
 
+	/**
+	 * Say the status of any buffers, particularly in output buffers of 
+	 * data blocks, but can add bespoke info for other internal buffers
+	 * for some processes. 
+	 * @param message
+	 * @param sayEmpties include info even if a buffer is empty. 
+	 */
+	public void dumpBufferStatus(String message, boolean sayEmpties) {
+		ArrayList<PamDataBlock> outputs = getOutputDataBlocks();
+		try {
+			for (PamDataBlock output : outputs) {
+				output.dumpBufferStatus(message, sayEmpties);
+			}
+		}
+		catch (Exception e) {
+			System.err.println("Error dumping buffer data from process " + getProcessName());
+			e.printStackTrace();
+		}
+	}
+
 }

src/PamguardMVC/PamRawDataBlock.java

@@ -145,6 +145,17 @@ public class PamRawDataBlock extends AcousticDataBlock<RawDataUnit> {
 		}
 	}
 	
+	/**
+	 * Reset data integrity checking counters. 
+	 */
+	public void reset() {
+		prevChannelSample = new long[PamConstants.MAX_CHANNELS];
+		summaryTotals = new double[PamConstants.MAX_CHANNELS];
+		summaryTotals2 = new double[PamConstants.MAX_CHANNELS];
+		summaryMaxVal = new double[PamConstants.MAX_CHANNELS];
+		summaryCount = new int[PamConstants.MAX_CHANNELS];
+	}
+	
 	@Override
 	public void addPamData(RawDataUnit pamDataUnit) {
 		/*

src/PamguardMVC/ThreadedObserver.java

@@ -444,6 +444,7 @@ public class ThreadedObserver implements PamObserver {
 				else {
 					emptyRead = false;
 					int lc=0;
+					ObservedObject observedObject;
 					while (!toDoList.isEmpty()) {
 
 //						if (stopFlag) {
@@ -458,11 +459,21 @@ public class ThreadedObserver implements PamObserver {
 						
 
 						// get the first object, send it for processing and then remove from the list
-						ObservedObject observedObject = toDoList.get(0);
-						performAction(observedObject);
 						synchronized(synchLock) {
-							toDoList.remove(0);
+							if (toDoList.size() > 0) { 
+								observedObject = toDoList.remove(0);
+							}
+							else {
+								break;
+							}
 						}
+						// need to do this bit outside of the synch block. 
+						performAction(observedObject);
+//						synchronized(synchLock) {
+//							if (toDoList.size() > 0) { // list may have been cleared during a shut down. 
+//								toDoList.remove(0);
+//							}
+//						}
 					}
 				}
 			}			
@@ -525,4 +536,21 @@ public class ThreadedObserver implements PamObserver {
 		}
 	}
 
+
+	public void clearEverything() {
+		synchronized (synchLock) { 
+			System.out.printf("Clearing %d objects from todo list in %s\n", toDoList.size(), singleThreadObserver.getObserverName());
+			toDoList.clear();
+		}		
+	}
+
+	public void dumpBufferStatus(String message, boolean sayEmpties) {
+		int n = toDoList.size();
+		if (sayEmpties == false && n == 0) {
+			return;
+		}
+		String name = singleThreadObserver.getObserverName();
+		System.out.printf("Threaded observer %s has %d objects in queue\n", name, n);
+	}
+
 }

src/binaryFileStorage/BinaryOutputStream.java

@@ -49,7 +49,7 @@ public class BinaryOutputStream {
 	
 	private DataOutputStream noiseOutputStream;
 
-	private int storedObjects;
+	private int storedObjects, storedNoiseCount;
 
 	private String mainFileName, indexFileName;
 
@@ -219,6 +219,7 @@ public class BinaryOutputStream {
 		else {
 			noiseOutputStream = null;
 		}
+		storedNoiseCount = 0;
 		return true;
 	}
 
@@ -450,6 +451,7 @@ public class BinaryOutputStream {
 		footer.setHighestUID(parentDataBlock.getUidHandler().getCurrentUID());
 		boolean ok = footer.writeFooter(dataOutputStream, BinaryStore.getCurrentFileFormat());
 		if (noiseOutputStream != null) {
+			footer.setnObjects(storedNoiseCount);
 			ok &= footer.writeFooter(noiseOutputStream, BinaryStore.getCurrentFileFormat());
 		}
 		lastObjectType = BinaryTypes.FILE_FOOTER;
@@ -487,12 +489,20 @@ public class BinaryOutputStream {
 //	}
 	
 	public synchronized boolean storeData(int objectId, DataUnitBaseData baseData, BinaryObjectData binaryObjectData) {
+		boolean ok;
 		if (objectId == BinaryTypes.BACKGROUND_DATA & noiseOutputStream != null) {
-			return storeData(noiseOutputStream, objectId, baseData, binaryObjectData);
+			ok = storeData(noiseOutputStream, objectId, baseData, binaryObjectData);
+			if (ok) {
+				storedNoiseCount++;
+			}
 		}
 		else {
-			return storeData(dataOutputStream, objectId, baseData, binaryObjectData);
+			ok = storeData(dataOutputStream, objectId, baseData, binaryObjectData);
+			if (ok) {
+				storedObjects++;
+			}
 		}
+		return ok;
 	}
 	/**
 	 * Writes data to a file. Note that the length of data may be greater than
@@ -564,7 +574,6 @@ public class BinaryOutputStream {
 			return false;
 		}
 
-		storedObjects++;
 
 
 		return true;

src/clickTrainDetector/clickTrainAlgorithms/mht/MHTClickTrainAlgorithm.java

@@ -45,6 +45,7 @@ public class MHTClickTrainAlgorithm implements ClickTrainAlgorithm, PamSettings
 	
 	public static final String MHT_NAME = "MHT detector"; 
 	
+	
 	/**
 	 * Reference to the click train control. 
 	 */
@@ -354,6 +355,7 @@ public class MHTClickTrainAlgorithm implements ClickTrainAlgorithm, PamSettings
 		TrackBitSet trackBitSet;
 		TrackDataUnits trackUnits;
 		if (nTracks>0) Debug.out.println("-------------------Grab Done Trains---------------");
+		try {
 		for (int i =0; i<nTracks; i++) {
 			trackBitSet=mhtKernal.getConfirmedTrack(i);
 			Debug.out.println("MHTAlgorithm: Confirmed Track Grab: No. " + MHTKernel.getTrueBitCount(trackBitSet.trackBitSet)  + " flag: " + trackBitSet.flag + "  chi2: " +trackBitSet.chi2Track.getChi2()); 
@@ -379,6 +381,10 @@ public class MHTClickTrainAlgorithm implements ClickTrainAlgorithm, PamSettings
 			trackCount++;
 			saveClickTrain(trackUnits, trackBitSet.chi2Track.getMHTChi2Info());
 		}
+		}
+		catch (Exception e) {
+			System.out.printf("Handled MHTClickTrainAlgorithm Exception %s in grabDoneTrains: %s\n", e.getClass().getSimpleName(), e.getMessage());
+		}
 		
 		if (nTracks>0) Debug.out.println("-------------------------------------------------");
 
@@ -415,12 +421,19 @@ public class MHTClickTrainAlgorithm implements ClickTrainAlgorithm, PamSettings
 		return mhtGUI;
 	}
 
+	Thread previousThread = null;
 	/**
 	 * Update the algorithm
 	 * @param flag- flag indicating the update type. 
 	 */
-	public void update(int flag, Object info) {
+	public synchronized void update(int flag, Object info) {
 
+		if (Thread.currentThread() != previousThread) {
+			// see flag id constants in ClickTrianControl
+			System.out.printf("Thread change to %s in MHTClicktrainAlgorithm.update flag %d, object %s\n", 
+					Thread.currentThread().toString(), flag, info);
+			previousThread = Thread.currentThread();
+		}
 		switch (flag) {
 		case ClickTrainControl.PROCESSING_START:
 			//make sure the kernel is cleared before processing

src/clickTrainDetector/clickTrainAlgorithms/mht/MHTKernel.java

@@ -3,6 +3,7 @@ package clickTrainDetector.clickTrainAlgorithms.mht;
 import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.Collections;
+import java.util.Comparator;
 
 import PamUtils.PamArrayUtils;
 import PamguardMVC.debug.Debug;
@@ -285,28 +286,39 @@ public class MHTKernel<T> {
 			BitSet currentBitSet;
 			MHTChi2<T> mhtChi2;
 			int index; 
-			for (int i=0; i<possibleTracks.size(); i++) {
+			synchronized(trackSynchronisation) {
+				try {
+				for (int i=0; i<possibleTracks.size(); i++) {
 
-				currentBitSet=possibleTracks.get(i).trackBitSet;
+					currentBitSet=possibleTracks.get(i).trackBitSet;
 
-				//index is the total detection count-1; 
-				index=kcount-1; 
+					//index is the total detection count-1; 
+					index=kcount-1; 
 
-				//now add both a true and false for the data unit to be in this possibility. 
-				currentBitSet.set(index, true);
-				mhtChi2=possibleTracks.get(i).chi2Track.cloneMHTChi2(); 
+					//now add both a true and false for the data unit to be in this possibility. 
+					currentBitSet.set(index, true);
+					mhtChi2=possibleTracks.get(i).chi2Track.cloneMHTChi2(); 
 
-				newPossibilities.add(new TrackBitSet(currentBitSet, mhtChi2)); 
+					newPossibilities.add(new TrackBitSet(currentBitSet, mhtChi2)); 
 
-				//add a coast to the possibility
-				currentBitSet=(BitSet) currentBitSet.clone(); 
-				currentBitSet.set(index, false);
-				//currentBitSet.set(currentBitSet.size(), true);
-				//add new chi2 value -  need to clone this time. 
-				mhtChi2=possibleTracks.get(i).chi2Track.cloneMHTChi2(); 
+					//add a coast to the possibility
+					currentBitSet=(BitSet) currentBitSet.clone(); 
+					currentBitSet.set(index, false);
+					//currentBitSet.set(currentBitSet.size(), true);
+					//add new chi2 value -  need to clone this time. 
+					/*
+					 * This line can throw an error due to poor sunchronisation if 
+					 * the list is emptied from a different thread. 
+					 */
+					mhtChi2=possibleTracks.get(i).chi2Track.cloneMHTChi2(); 
 
-				//added the cloned bitset to not mess up references 
-				newPossibilities.add(new TrackBitSet(currentBitSet, mhtChi2)); 
+					//added the cloned bitset to not mess up references 
+					newPossibilities.add(new TrackBitSet(currentBitSet, mhtChi2)); 
+				}
+				}
+				catch (Exception e) {
+					System.out.printf("*******  MHTKernel Exception %s in growProbMatrix: %s\n", e.getClass().getSimpleName(), e.getMessage());
+				}
 			}
 		}
 
@@ -342,10 +354,21 @@ public class MHTKernel<T> {
 			//first sort the tracks by increasing chi2 values. 
 			//sort the possible tracks by chi2 values
 			//now sort the chi2 values so they correspond to the track list.  
-			Collections.sort(newPossibleTracks, (left, right)->{
-				//Note- this is definitely in the correct order
-				return Double.compare(left.chi2Track.getChi2(), right.chi2Track.getChi2());
-			});		
+//			Collections.sort(newPossibleTracks, (left, right)->{
+//				//Note- this is definitely in the correct order
+//				return Double.compare(left.chi2Track.getChi2(), right.chi2Track.getChi2());
+//			});		
+			try {
+			Collections.sort(newPossibleTracks, new Comparator<TrackBitSet>() {
+				@Override
+				public int compare(TrackBitSet left, TrackBitSet right) {
+					return Double.compare(left.chi2Track.getChi2(), right.chi2Track.getChi2());
+				}
+			});
+			}
+			catch (Exception e) {
+				System.out.printf("Handled  MHTKernel Exception %s in pruneProbMatrix: %s\n", e.getClass().getSimpleName(), e.getMessage());
+			}
 
 			//			for (int i=0; i<newPossibleTracks.size(); i++) {
 			//				System.out.print("Possibility chi2: " + i +  "  " + String.format("%.3f", newPossibleTracks.get(i).chi2Track.getChi2()));

src/decimator/DecimatorProcessW.java

@@ -38,8 +38,10 @@ public class DecimatorProcessW extends PamProcess {
 
 	@Override
 	public void pamStart() {
-		// TODO Auto-generated method stub
-
+		outputDataBlock.reset();
+		if (decimatorWorker != null) {
+			decimatorWorker.reset();
+		}
 	}
 
 	@Override

src/decimator/DecimatorWorker.java

@@ -77,6 +77,13 @@ public class DecimatorWorker {
 		createFilters();
 	}
 	
+	/**
+	 * Reset all counters and output buffers. 
+	 */
+	public void reset() {
+		createFilters();
+	}
+	
 	/**
 	 * Make the decimator filters. If reducing frequency, then the filter
 	 * is applied before decimation (obviously!) so is set up based on the 

src/fftManager/PamFFTProcess.java

@@ -124,6 +124,7 @@ public class PamFFTProcess extends PamProcess {
 
 	public synchronized void setupFFT() {
 		
+		System.out.println("In call to setupFFT in " + getProcessName());
 		// need to find the existing source data block and remove from observing it.
 		// then find the new one and subscribe to that instead. 
 		channelCounts = new int[PamConstants.MAX_CHANNELS];
@@ -390,28 +391,32 @@ public class PamFFTProcess extends PamProcess {
 				}
 			}
 		}
+		TempOutputStore[] oldStores = tempStores;
 		if (iChan == PamUtils.getHighestChannel(fftParameters.channelMap)) {
 			// time to empty the stores - assume they all have the same
 			// amount of data 
 			int[] chanList = PamUtils.getChannelArray(fftParameters.channelMap);
 			try {
-			int n = tempStores[iChan].getN();
-			for (int iF = 0; iF < n; iF++) {
-				for (int iC = 0; iC < chanList.length; iC++) {
-//					pu = tempStores[chanList[iC]].get(iF);
-					try {
-					outputData.addPamData(tempStores[chanList[iC]].get(iF));
+				int n = tempStores[iChan].getN();
+				for (int iF = 0; iF < n; iF++) {
+					for (int iC = 0; iC < chanList.length; iC++) {
+						//					pu = tempStores[chanList[iC]].get(iF);
+						try {
+							outputData.addPamData(tempStores[chanList[iC]].get(iF));
+						}
+						catch (ArrayIndexOutOfBoundsException e) {
+							//						e.printStackTrace();
+							System.err.printf("%s.newData: %s Store %s (was %s) iC: %d of %d iF: %d of %d\n", 
+									this.getPamControlledUnit().getUnitName(), e.getMessage(), 
+									tempStores[chanList[iC]], oldStores[chanList[iC]],
+									iC, chanList.length, iF, n);
+						}
+						//					outputData.addPamData(null);
 					}
-					catch (ArrayIndexOutOfBoundsException e) {
-//						e.printStackTrace();
-						System.err.println("PAMFFTProcess.newData: " + e.getMessage() + " " + this.getPamControlledUnit().getUnitName() + " iC: " + iC +  " iF: " + iF);
-					}
-//					outputData.addPamData(null);
 				}
-			}
-			for (int iC = 0; iC < chanList.length; iC++) {
-				tempStores[chanList[iC]].clearStore();
-			}
+				for (int iC = 0; iC < chanList.length; iC++) {
+					tempStores[chanList[iC]].clearStore();
+				}
 			}
 			catch (Exception e) {
 				e.printStackTrace();
@@ -539,6 +544,25 @@ public class PamFFTProcess extends PamProcess {
 		return new ArrayList<Class<? extends PamDataUnit>>(Arrays.asList(RawDataUnit.class));
 	}
 
+	@Override
+	public synchronized void dumpBufferStatus(String message, boolean sayEmpties) {
+		
+		super.dumpBufferStatus(message, sayEmpties);
+		int nTemp = 0;
+		if (tempStores != null) {
+			nTemp = tempStores.length;
+		}
+		for (int i = 0; i < nTemp; i++) {
+			if (tempStores[i] == null) {
+				continue;
+			}
+			int n = tempStores[i].tempUnits.size();
+			if (n > 0 || sayEmpties) {
+				System.out.printf("FFT %s temp store %d has %d datas\n", getProcessName(), i, n);
+			}
+		}
+	}
+
 //	@Override
 //	public boolean requestOfflineData(PamDataBlock dataBlock, long startMillis,
 //			long endMillis) {

src/pamScrollSystem/DataLoadQueData.java

@@ -1,5 +1,6 @@
 package pamScrollSystem;
 
+import PamUtils.PamCalendar;
 import PamguardMVC.PamDataBlock;
 
 /**
@@ -68,6 +69,14 @@ public class DataLoadQueData {
 	public void setDataEnd(long dataEnd) {
 		this.dataEnd = dataEnd;
 	}
+
+
+	@Override
+	public String toString() {
+		String str = String.format("%s %s - %s", pamDataBlock.getLongDataName(),
+				PamCalendar.formatDBDateTime(dataStart),PamCalendar.formatDBDateTime(dataEnd));
+		return str;
+	}
 	
 	
 }