Functions for better restarting after buffer overflow.

src/Acquisition/AcquisitionProcess.java

@@ -211,11 +211,11 @@ public class AcquisitionProcess extends PamProcess implements DataInputStore {
 
 		if (systemPrepared == false) return;
 
+		newDataQueue.clearList(); // clear this first to make sure nothing new comes in. 
+
 		//		before starting, clear all old data
 		rawDataBlock.clearAll();
 		
-		newDataQueue.clearList();
-		
 		sampleRateErrorFilter.prepareFilter();
 		totalExtraSamples = 0;
 		millisecondSampleOffset = 0;
@@ -302,6 +302,7 @@ public class AcquisitionProcess extends PamProcess implements DataInputStore {
 		// called by PamController.
 		// stop the running system - not the selected system since
 		// this may have changed
+				
 		restartTimer.stop();
 //		stallCheckTimer.stop();
 		pamStop("");
@@ -471,6 +472,9 @@ public class AcquisitionProcess extends PamProcess implements DataInputStore {
 	 * have been emptied and processing has stopped
 	 */
 	protected void pamHasStopped() {
+		
+		newDataQueue.clearList(); // clear this first to make sure nothing new comes in. 
+		
 		if (runningSystem == null){
 			runningSystem = acquisitionControl.findDaqSystem(null);
 		}
@@ -650,9 +654,10 @@ public class AcquisitionProcess extends PamProcess implements DataInputStore {
 
 			restartTimer.stop();
 
-			PamController.getInstance().pamStop();
+			PamController.getInstance().restartPamguard();
-
+//			PamController.getInstance().pamStop();
-			PamController.getInstance().pamStart(false);
+//
+//			PamController.getInstance().pamStart(false);
 
 		}
 

src/PamController/PamController.java

@@ -129,6 +129,10 @@ public class PamController implements PamControllerInterface, PamSettings {
 	public static final int PAM_MAPMAKING = 7;
 	public static final int PAM_OFFLINETASK = 8;
 	
+	public static final int BUTTON_START = 1;
+	public static final int BUTTON_STOP = 2;
+	private volatile int lastStartStopButton = 0;
+
 	// status' for RunMode = RUN_PAMVIEW
 	public static final int PAM_LOADINGDATA = 2;
 
@@ -240,6 +244,9 @@ public class PamController implements PamControllerInterface, PamSettings {
 	private Thread statusCheckThread;
 	private WaitDetectorThread detectorEndThread;
 	private boolean firstDataLoadComplete;
+	// keep a track of the total number of times PAMGuard is started for debug purposes. 
+	private int nStarts;
+	private RestartRunnable restartRunnable;
 
 
 	private PamController(int runMode, Object object) {
@@ -1045,8 +1052,45 @@ public class PamController implements PamControllerInterface, PamSettings {
 	 */
 	public void restartPamguard() {
 		pamStop();
-		startLater();		
+		
+		/*
+		 *  launch a restart thread, that won't do ANYTHING until
+		 *  PAMGuard is really idle and buffers are cleared. Can only
+		 *  have one of these at a time ! 
+		 */
+		if (restartRunnable != null) {
+			System.out.println("Warning !!!! PAMGuard is already trying to restart!");
+			return;
 		}
+		restartRunnable = new RestartRunnable();
+		Thread restartThread = new Thread(restartRunnable, "RestartPAMGuard Thread");
+		restartThread.run();
+	}
+	
+	private class RestartRunnable implements Runnable {
+
+		@Override
+		public void run() {
+			long t1 = System.currentTimeMillis();
+			while (getPamStatus() != PAM_IDLE) {
+				try {
+					Thread.sleep(200);
+				} catch (InterruptedException e) {
+
+				}
+			}
+			long t2 = System.currentTimeMillis();
+			restartRunnable = null;
+			System.out.printf("PAMGuard safe to restart after %d milliseconds\n", t2-t1);
+			startLater(false);	
+			
+		}
+		
+	}
+	
+	
+	
+	
 	/**
 	 * calls pamStart using the SwingUtilities
 	 * invokeLater command to start PAMGAURD 
@@ -1078,9 +1122,15 @@ public class PamController implements PamControllerInterface, PamSettings {
 
 		@Override
 		public void run() {
+			/*
+			 *  do a final check that the stop button hasn't been pressed - can arrive a bit
+			 *  late if the system was continually restarting.  
+			 */
+			if (lastStartStopButton != BUTTON_STOP) {
 				pamStart(saveSettings);
 			}
 		}
+	}
 
 	/**
 	 * calls pamStop using the SwingUtilities
@@ -1103,6 +1153,26 @@ public class PamController implements PamControllerInterface, PamSettings {
 		}
 	}
 
+	/**
+	 * Called from the start button. A little book keeping 
+	 * to distinguish this from automatic starts / restarts
+	 * @return true if started.
+	 */
+	@Override
+	public boolean manualStart() {
+		lastStartStopButton = BUTTON_START;
+		return pamStart();
+	}
+	
+	/**
+	 * Called from the stop button. A little book keeping 
+	 * to distinguish this from automatic starts / restarts
+	 */
+	@Override
+	public void manualStop() {
+		lastStartStopButton = BUTTON_STOP;
+		pamStop();
+	}
 
 	/**
 	 * Start PAMGUARD. This function also gets called from the 
@@ -1193,6 +1263,12 @@ public class PamController implements PamControllerInterface, PamSettings {
 			saveSettings(PamCalendar.getSessionStartTime());
 		}
 
+		if (++nStarts > 1) {
+			// do this here - all processses should have reset buffers to start again by now. 
+			String msg = String.format("Starting PAMGuard go %d", nStarts);
+			dumpBufferStatus(msg, false);
+		}
+
 		StorageOptions.getInstance().setBlockOptions();
 
 		t1 = System.currentTimeMillis();
@@ -1254,6 +1330,7 @@ public class PamController implements PamControllerInterface, PamSettings {
 			}
 		}
 		
+		dumpBufferStatus("In stopping", false);
 		/*
 		 *  now launch another thread to wait for everything to have stopped, but
 		 *  leave this function so that AWT is released and graphics can update, the
@@ -1281,9 +1358,11 @@ public class PamController implements PamControllerInterface, PamSettings {
 				long t2 = System.currentTimeMillis();
 				if (t2 - t1 > 5000) {
 					System.out.printf("Stopping, but stuck in loop for CheckRunStatus for %3.1fs\n", (double) (t2-t1)/1000.);
+					dumpBufferStatus("Stopping stuck in loop", false);
+					break; // crap out anyway.
 				}
 				try {
-					Thread.sleep(10);
+					Thread.sleep(100);
 				} catch (InterruptedException e) {
 					e.printStackTrace();
 				}
@@ -1294,19 +1373,41 @@ public class PamController implements PamControllerInterface, PamSettings {
 		
 	}
 	
+	/**
+	 * Look in every data block, particularly threaded ones, and dump
+	 * the buffer status. This will have to go via PamProcess so that 
+	 * additional information can be added from any processes that 
+	 * hold additional data in other internal buffers. 
+	 * @param message Message to print prior to dumping buffers for debug. 
+	 * @param sayEmpties dump info even if a buffer is empty (otherwise, only ones that have stuff still)
+	 */
+	public void dumpBufferStatus(String message, boolean sayEmpties) {
+		System.out.println("**** Dumping process buffer status: " + message);
+		ArrayList<PamControlledUnit> pamControlledUnits = pamConfiguration.getPamControlledUnits();
+		for (PamControlledUnit aUnit : pamControlledUnits) {
+			int numProcesses = aUnit.getNumPamProcesses();
+			for (int i=0; i<numProcesses; i++) {
+				PamProcess aProcess = aUnit.getPamProcess(i);
+				aProcess.dumpBufferStatus(message, sayEmpties);
+			}
+		}
+		System.out.println("**** End of process buffer dump: " + message);
+	}
+	
 	/**
 	 * Called once the detectors have actually stopped and puts a few finalising 
 	 * functions into the AWT thread. 
 	 */
 	private void finishStopping() {
 		detectorEndThread = null;
-		SwingUtilities.invokeLater(new Runnable() {
+		// this was never getting invoked for some reason. 
-			
+//		SwingUtilities.invokeLater(new Runnable() {
-			@Override
+//			
-			public void run() {
+//			@Override
+//			public void run() {
 				pamStopped();
-			}
+//			}
-		});
+//		});
 	}
 		
 	
@@ -1320,6 +1421,8 @@ public class PamController implements PamControllerInterface, PamSettings {
 		 * it is necessary to make sure that all internal datablock 
 		 * buffers have had time to empty.
 		 */
+		System.out.println("Arrived in PamStopped() in thread " + Thread.currentThread().toString());
+		
 		ArrayList<PamControlledUnit> pamControlledUnits = pamConfiguration.getPamControlledUnits();
 		
 		if (PamModel.getPamModel().isMultiThread()) {
@@ -1328,6 +1431,7 @@ public class PamController implements PamControllerInterface, PamSettings {
 			}
 		}
 		setPamStatus(PAM_IDLE);
+		dumpBufferStatus("In pamStopped, now idle", true);
 
 		// wait here until the status has changed to Pam_Idle, so that we know
 		// that we've really finished processing all data
@@ -2054,6 +2158,7 @@ public class PamController implements PamControllerInterface, PamSettings {
 			statusWarning.setWarningMessage(warningMessage);
 			statusWarning.setWarnignLevel(1);
 			warningSystem.addWarning(statusWarning);
+//			System.out.println(warningMessage);
 		}
 	}
 

src/PamController/PamControllerInterface.java

@@ -459,6 +459,17 @@ public interface PamControllerInterface {
 	 */
 	public void pamClose();
 
+	/**
+	 * Start function called from button to do a bit of extra book keeping
+	 * @return
+	 */
+	public boolean manualStart();
+
+	/**
+	 * Stop function called from button to do a bit of extra book keeping
+	 */
+	public void manualStop();
+	
 	//public void controllerAddFileMenuItem();
 
 }

src/PamView/PamGui.java

@@ -1268,7 +1268,7 @@ public class PamGui extends PamView implements WindowListener, PamSettings {
 
 	class menuPamStart implements ActionListener {
 		public void actionPerformed(ActionEvent ev){
-			pamControllerInterface.pamStart();
+			pamControllerInterface.manualStart();
 
 		}
 	}
@@ -1285,7 +1285,7 @@ public class PamGui extends PamView implements WindowListener, PamSettings {
 
 	class menuPamStop implements ActionListener {
 		public void actionPerformed(ActionEvent ev){
-			pamControllerInterface.pamStop();
+			pamControllerInterface.manualStop();
 			//			enableLoggingMenu();		
 		}
 	}

src/PamguardMVC/PamDataBlock.java

@@ -4286,4 +4286,23 @@ public class PamDataBlock<Tunit extends PamDataUnit> extends PamObservable {
 		inputEl.setAttribute("Channels", String.format("0x%X", getChannelMap()));
 		return inputEl;
 	}
+
+	/**
+	 * Look in every data block, particularly threaded ones, and dump
+	 * the buffer status. This will have to go via PamProcess so that 
+	 * additional information can be added from any processes that 
+	 * hold additional data in other internal buffers. 
+	 * @param message Message to print prior to dumping buffers for debug. 
+	 * @param sayEmpties dump info even if a buffer is empty (otherwise, only ones that have stuff still)
+	 */
+	public void dumpBufferStatus(String message, boolean sayEmpties) {
+		int nObs = countObservers();
+		for (int i = 0; i < nObs; i++) {
+			PamObserver obs = getPamObserver(i);
+			if (obs instanceof ThreadedObserver) {
+				ThreadedObserver tObs = (ThreadedObserver) obs;
+				tObs.dumpBufferStatus(message, sayEmpties);
+			}
+		}
+	}
 }

src/PamguardMVC/PamObservable.java

@@ -208,6 +208,14 @@ public class PamObservable {//extends PanelOverlayDraw {
 			if (System.currentTimeMillis() - startTime > timeOutms) {
 				// have taken too long, so return that we've failed. 
 				System.out.println("Wait timeout in threaded observer");
+				// and clear everything that's left. 
+				for (int i = 0; i < pamObservers.size(); i++) {
+					pamObserver = pamObservers.get(i);
+					if (pamObserver.getClass() == ThreadedObserver.class) {
+						threadedObserver = (ThreadedObserver) pamObserver;
+						threadedObserver.clearEverything();
+					}
+				}
 				return false;
 			}
 			try {

src/PamguardMVC/PamProcess.java

@@ -767,7 +767,7 @@ abstract public class PamProcess implements PamObserver, ProcessAnnotator {
 		}
 	});
 
-	private int lastSourceNotificationType;
+	private volatile int lastSourceNotificationType;
 
 	private Object lastSourceNotificationObject;
 	
@@ -1065,4 +1065,24 @@ abstract public class PamProcess implements PamObserver, ProcessAnnotator {
 		return lastSourceNotificationObject;
 	}
 
+	/**
+	 * Say the status of any buffers, particularly in output buffers of 
+	 * data blocks, but can add bespoke info for other internal buffers
+	 * for some processes. 
+	 * @param message
+	 * @param sayEmpties include info even if a buffer is empty. 
+	 */
+	public void dumpBufferStatus(String message, boolean sayEmpties) {
+		ArrayList<PamDataBlock> outputs = getOutputDataBlocks();
+		try {
+			for (PamDataBlock output : outputs) {
+				output.dumpBufferStatus(message, sayEmpties);
+			}
+		}
+		catch (Exception e) {
+			System.err.println("Error dumping buffer data from process " + getProcessName());
+			e.printStackTrace();
+		}
+	}
+
 }

src/PamguardMVC/PamRawDataBlock.java

@@ -145,6 +145,17 @@ public class PamRawDataBlock extends AcousticDataBlock<RawDataUnit> {
 		}
 	}
 	
+	/**
+	 * Reset data integrity checking counters. 
+	 */
+	public void reset() {
+		prevChannelSample = new long[PamConstants.MAX_CHANNELS];
+		summaryTotals = new double[PamConstants.MAX_CHANNELS];
+		summaryTotals2 = new double[PamConstants.MAX_CHANNELS];
+		summaryMaxVal = new double[PamConstants.MAX_CHANNELS];
+		summaryCount = new int[PamConstants.MAX_CHANNELS];
+	}
+	
 	@Override
 	public void addPamData(RawDataUnit pamDataUnit) {
 		/*

src/PamguardMVC/ThreadedObserver.java

@@ -444,6 +444,7 @@ public class ThreadedObserver implements PamObserver {
 				else {
 					emptyRead = false;
 					int lc=0;
+					ObservedObject observedObject;
 					while (!toDoList.isEmpty()) {
 
 //						if (stopFlag) {
@@ -458,11 +459,21 @@ public class ThreadedObserver implements PamObserver {
 						
 
 						// get the first object, send it for processing and then remove from the list
-						ObservedObject observedObject = toDoList.get(0);
-						performAction(observedObject);
 						synchronized(synchLock) {
-							toDoList.remove(0);
+							if (toDoList.size() > 0) { 
+								observedObject = toDoList.remove(0);
 							}
+							else {
+								break;
+							}
+						}
+						// need to do this bit outside of the synch block. 
+						performAction(observedObject);
+//						synchronized(synchLock) {
+//							if (toDoList.size() > 0) { // list may have been cleared during a shut down. 
+//								toDoList.remove(0);
+//							}
+//						}
 					}
 				}
 			}			
@@ -525,4 +536,21 @@ public class ThreadedObserver implements PamObserver {
 		}
 	}
 
+
+	public void clearEverything() {
+		synchronized (synchLock) { 
+			System.out.printf("Clearing %d objects from todo list in %s\n", toDoList.size(), singleThreadObserver.getObserverName());
+			toDoList.clear();
+		}		
+	}
+
+	public void dumpBufferStatus(String message, boolean sayEmpties) {
+		int n = toDoList.size();
+		if (sayEmpties == false && n == 0) {
+			return;
+		}
+		String name = singleThreadObserver.getObserverName();
+		System.out.printf("Threaded observer %s has %d objects in queue\n", name, n);
+	}
+
 }

src/clickTrainDetector/clickTrainAlgorithms/mht/MHTClickTrainAlgorithm.java

@@ -415,12 +415,19 @@ public class MHTClickTrainAlgorithm implements ClickTrainAlgorithm, PamSettings
 		return mhtGUI;
 	}
 
+	Thread previousThread = null;
 	/**
 	 * Update the algorithm
 	 * @param flag- flag indicating the update type. 
 	 */
 	public void update(int flag, Object info) {
 
+		if (Thread.currentThread() != previousThread) {
+			// see flag id constants in ClickTrianControl
+			System.out.printf("Thread change to %s in MHTClicktrainAlgorithm.update flag %d, object %s\n", 
+					Thread.currentThread().toString(), flag, info);
+			previousThread = Thread.currentThread();
+		}
 		switch (flag) {
 		case ClickTrainControl.PROCESSING_START:
 			//make sure the kernel is cleared before processing

src/clickTrainDetector/clickTrainAlgorithms/mht/MHTKernel.java

@@ -285,6 +285,7 @@ public class MHTKernel<T> {
 			BitSet currentBitSet;
 			MHTChi2<T> mhtChi2;
 			int index; 
+			synchronized(trackSynchronisation) {
 				for (int i=0; i<possibleTracks.size(); i++) {
 
 					currentBitSet=possibleTracks.get(i).trackBitSet;
@@ -303,12 +304,17 @@ public class MHTKernel<T> {
 					currentBitSet.set(index, false);
 					//currentBitSet.set(currentBitSet.size(), true);
 					//add new chi2 value -  need to clone this time. 
+					/*
+					 * This line can throw an error due to poor sunchronisation if 
+					 * the list is emptied from a different thread. 
+					 */
 					mhtChi2=possibleTracks.get(i).chi2Track.cloneMHTChi2(); 
 
 					//added the cloned bitset to not mess up references 
 					newPossibilities.add(new TrackBitSet(currentBitSet, mhtChi2)); 
 				}
 			}
+		}
 
 		long time2 = System.currentTimeMillis(); 
 

src/decimator/DecimatorProcessW.java

@@ -38,8 +38,10 @@ public class DecimatorProcessW extends PamProcess {
 
 	@Override
 	public void pamStart() {
-		// TODO Auto-generated method stub
+		outputDataBlock.reset();
-
+		if (decimatorWorker != null) {
+			decimatorWorker.reset();
+		}
 	}
 
 	@Override

src/decimator/DecimatorWorker.java

@@ -77,6 +77,13 @@ public class DecimatorWorker {
 		createFilters();
 	}
 	
+	/**
+	 * Reset all counters and output buffers. 
+	 */
+	public void reset() {
+		createFilters();
+	}
+	
 	/**
 	 * Make the decimator filters. If reducing frequency, then the filter
 	 * is applied before decimation (obviously!) so is set up based on the 

src/fftManager/PamFFTProcess.java

@@ -124,6 +124,7 @@ public class PamFFTProcess extends PamProcess {
 
 	public synchronized void setupFFT() {
 		
+		System.out.println("In call to setupFFT in " + getProcessName());
 		// need to find the existing source data block and remove from observing it.
 		// then find the new one and subscribe to that instead. 
 		channelCounts = new int[PamConstants.MAX_CHANNELS];
@@ -390,6 +391,7 @@ public class PamFFTProcess extends PamProcess {
 				}
 			}
 		}
+		TempOutputStore[] oldStores = tempStores;
 		if (iChan == PamUtils.getHighestChannel(fftParameters.channelMap)) {
 			// time to empty the stores - assume they all have the same
 			// amount of data 
@@ -398,15 +400,18 @@ public class PamFFTProcess extends PamProcess {
 				int n = tempStores[iChan].getN();
 				for (int iF = 0; iF < n; iF++) {
 					for (int iC = 0; iC < chanList.length; iC++) {
-//					pu = tempStores[chanList[iC]].get(iF);
+						//					pu = tempStores[chanList[iC]].get(iF);
 						try {
 							outputData.addPamData(tempStores[chanList[iC]].get(iF));
 						}
 						catch (ArrayIndexOutOfBoundsException e) {
-//						e.printStackTrace();
+							//						e.printStackTrace();
-						System.err.println("PAMFFTProcess.newData: " + e.getMessage() + " " + this.getPamControlledUnit().getUnitName() + " iC: " + iC +  " iF: " + iF);
+							System.err.printf("%s.newData: %s Store %s (was %s) iC: %d of %d iF: %d of %d\n", 
+									this.getPamControlledUnit().getUnitName(), e.getMessage(), 
+									tempStores[chanList[iC]], oldStores[chanList[iC]],
+									iC, chanList.length, iF, n);
 						}
-//					outputData.addPamData(null);
+						//					outputData.addPamData(null);
 					}
 				}
 				for (int iC = 0; iC < chanList.length; iC++) {
@@ -539,6 +544,25 @@ public class PamFFTProcess extends PamProcess {
 		return new ArrayList<Class<? extends PamDataUnit>>(Arrays.asList(RawDataUnit.class));
 	}
 
+	@Override
+	public synchronized void dumpBufferStatus(String message, boolean sayEmpties) {
+		
+		super.dumpBufferStatus(message, sayEmpties);
+		int nTemp = 0;
+		if (tempStores != null) {
+			nTemp = tempStores.length;
+		}
+		for (int i = 0; i < nTemp; i++) {
+			if (tempStores[i] == null) {
+				continue;
+			}
+			int n = tempStores[i].tempUnits.size();
+			if (n > 0 || sayEmpties) {
+				System.out.printf("FFT %s temp store %d has %d datas\n", getProcessName(), i, n);
+			}
+		}
+	}
+
 //	@Override
 //	public boolean requestOfflineData(PamDataBlock dataBlock, long startMillis,
 //			long endMillis) {