Big merge from doug (#139)

* binary store count

Fix issue in binary store object count

* update V10aa/other for testing

* update V10aa/other for testing

* reenable buffer dumping

* update V to 10ac for testing

* Additional diagnostics

Additional output of CPU usage for each module when stopping

* V 2.02.10ad for testing

Fixed issue of finding correct raw data block

* Tidying

Lots of GUI improvement and code tidying. Functionality to export
gzipped documents to reduce traffic.

* V 2.02.10ba for user testing

* Tidy up click selector

Improve layout and tips on dialog and improve logic for manual and
automatic event types.

* Menu tide up

* Update nilus maven for PAMGuard

* Fix reprocess choices

Make sure the choice to continue anyway is always present.

* Improve start of binary file timing

Code to better get binary files to start right on the hour when processing files offline rather than half a sec or so later.

* Fix early data discard

Fix issue in clip generator: when running very fast offline raw data being discarded before clips are generated. Changed threading model slightly and increased data keep time by 2x the thread jitter to try to avoid this.

* Update pom to JSerialCom 2.11.0

* V2.02.11e fix file start skip

Skipping start of files was causing click detector to not find clicks. Changed code so first seconds are still sent, but with data set to 0, rather than not sending data since that was causing sample counts in different bits of PAMGuard to get out of synch.

.classpath

@@ -5,7 +5,6 @@
 			<attribute name="optional" value="true"/>
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
-	</classpathentry>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-17">
 		<attributes>
 			<attribute name="maven.pomderived" value="true"/>

pom.xml

@@ -4,7 +4,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.pamguard</groupId>
   <artifactId>Pamguard</artifactId>
-  <version>2.02.11d</version>
+  <version>2.02.11e</version>
   <name>Pamguard</name>
   <description>Pamguard using Maven to control dependencies</description>
 	<url>www.pamguard.org</url>
@@ -579,7 +579,7 @@
 	<dependency>
 	    <groupId>com.fazecast</groupId>
 	    <artifactId>jSerialComm</artifactId>
-	    <version>2.5.3</version>
+	    <version>2.11.0</version>
 	</dependency>
 
 	<!-- https://mvnrepository.com/artifact/edu.emory.mathcs/JTransforms -->

src/Acquisition/FileInputSystem.java

@@ -15,6 +15,7 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Serializable;
 import java.text.DateFormat;
+import java.util.Arrays;
 import java.util.Calendar;
 import java.util.List;
 
@@ -1017,9 +1018,11 @@ public class FileInputSystem  extends DaqSystem implements ActionListener, PamSe
 						newDataUnit = new RawDataUnit(ms, 1 << ichan, totalSamples, newSamples);
 						newDataUnit.setRawData(doubleData[ichan]);
 
-						if (1000*(readFileSamples/sampleRate)>=fileInputParameters.skipStartFileTime) {
-							newDataUnits.addNewData(newDataUnit);
+						if (1000*(readFileSamples/sampleRate)<fileInputParameters.skipStartFileTime) {
+							// zero the data. Skipping it causes all the timing to screw up
+							Arrays.fill(doubleData[ichan], 0.);
 						}
+						newDataUnits.addNewData(newDataUnit);
 
 						// GetOutputDataBlock().addPamData(pamDataUnit);
 					}

src/PamController/PamController.java

@@ -2951,6 +2951,23 @@ public class PamController implements PamControllerInterface, PamSettings {
 		return pamConfiguration;
 	}
 
+	/**
+	 * Gets called on a timer when NOT processing from files. 
+	 * OR if processing files, gets called whenever the Calendar session start time or file time millis gets updated. 
+	 * @param timeInMillis
+	 */
+	public void updateMasterClock(long timeInMillis) {
+		/*
+		 *  this needs to notify binary stores that time has changed since the BS doesn't subscribe
+		 *  to anything, so doesn't get other clock updates. 
+		 */
+		ArrayList<PamControlledUnit> bs = findControlledUnits(BinaryStore.class);
+		for (PamControlledUnit aBS : bs) {
+			BinaryStore binStore = (BinaryStore) aBS;
+			binStore.getBinaryStoreProcess().checkFileTime(timeInMillis);
+		}
+	}
+
 
 
 }

src/PamController/PamguardVersionInfo.java

@@ -31,12 +31,12 @@ public class PamguardVersionInfo {
 	 * Version number, major version.minorversion.sub-release.
 	 * Note: can't go higher than sub-release 'f'
 	 */
-	static public final String version = "2.02.11d";
+	static public final String version = "2.02.11e";
 	
 	/**
 	 * Release date
 	 */
-	static public final String date = "30 May 2024";
+	static public final String date = "19 June 2024";
 	
 //	/**
 //	 * Release type - Beta or Core

src/PamController/fileprocessing/ReprocessChoiceDialog.java

@@ -4,6 +4,8 @@ import java.awt.BorderLayout;
 import java.awt.GridBagConstraints;
 import java.awt.GridBagLayout;
 import java.awt.Window;
+import java.awt.event.ActionEvent;
+import java.awt.event.ActionListener;
 import java.util.List;
 
 import javax.swing.ButtonGroup;
@@ -64,15 +66,28 @@ public class ReprocessChoiceDialog extends PamDialog {
 		List<ReprocessStoreChoice> userChoices = choiceSummary.getChoices();
 		choiceButtons = new JRadioButton[userChoices.size()];
 		ButtonGroup bg = new ButtonGroup();
+		SelAction selAction = new SelAction();
 		for (int i = 0; i < userChoices.size(); i++) {
 			ReprocessStoreChoice aChoice = userChoices.get(i);
 			choiceButtons[i] = new JRadioButton(aChoice.toString());
 			choiceButtons[i].setToolTipText(aChoice.getToolTip());
+			bg.add(choiceButtons[i]);
+			choiceButtons[i].addActionListener(selAction);
 			choicePanel.add(choiceButtons[i], c);
 			c.gridy++;
 		}
 		setDialogComponent(mainPanel);
 		getCancelButton().setVisible(false);
+		getOkButton().setEnabled(false);
+	}
+	
+	private class SelAction implements ActionListener {
+
+		@Override
+		public void actionPerformed(ActionEvent e) {
+			getOkButton().setEnabled(true);
+		}
+		
 	}
 	
 	public static ReprocessStoreChoice showDialog(Window parentFrame, StoreChoiceSummary choices) {

src/PamController/fileprocessing/ReprocessManager.java

@@ -62,7 +62,7 @@ public class ReprocessManager {
 		 */
 		boolean setupOK = setupInputStream(choiceSummary, choice);
 		
-		if (choice == ReprocessStoreChoice.DONTSSTART) {
+		if (choice == null || choice == ReprocessStoreChoice.DONTSSTART) {
 			return false;
 		}
 		
@@ -173,6 +173,8 @@ public class ReprocessManager {
 			return choiceSummary;
 		}
 
+		choiceSummary.addChoice(ReprocessStoreChoice.STARTNORMAL);
+		
 		ArrayList<PamControlledUnit> outputStores = PamController.getInstance().findControlledUnits(DataOutputStore.class, true);
 		boolean partStores = false; 
 		int nOutputStores = 0;

src/PamController/fileprocessing/StoreChoiceSummary.java

@@ -65,6 +65,9 @@ public class StoreChoiceSummary {
 	 * @param choice
 	 */
 	public void addChoice(ReprocessStoreChoice choice) {
+		if (choices.contains(choice)) {
+			return;
+		}
 		choices.add(choice);
 	}
 

src/PamUtils/PamCalendar.java

@@ -25,6 +25,7 @@ import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.time.LocalDateTime;
 import java.time.ZoneOffset;
+import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.Locale;
@@ -32,6 +33,7 @@ import java.util.TimeZone;
 
 import PamController.PamController;
 import PamUtils.time.CalendarControl;
+import binaryFileStorage.BinaryStore;
 
 
 /**
@@ -880,14 +882,18 @@ public class PamCalendar {
 	 */
 	public static void setSessionStartTime(long sessionStartTime) {
 		PamCalendar.sessionStartTime = sessionStartTime;
+		PamController.getInstance().updateMasterClock(getTimeInMillis());
 	}
 
 	/**
 	 * 
-	 * @param soundFileTimeMillis The start time of a sound file
+	 * Relative time within a sound file. This is always just added to sessionStartTime
+	 * to give an absolute time. 
+	 * @param soundFileTimeMillis The relative time of a sound file. 
 	 */
 	public static void setSoundFileTimeInMillis(long soundFileTimeMillis) {
 		PamCalendar.soundFileTimeInMillis = soundFileTimeMillis;
+		PamController.getInstance().updateMasterClock(getTimeInMillis());
 	}
 
 	/**

src/PamguardMVC/PamRawDataBlock.java

@@ -257,12 +257,13 @@ public class PamRawDataBlock extends AcousticDataBlock<RawDataUnit> {
 	synchronized public RawDataUnit[] getAvailableSamples(long startMillis, long durationMillis, int channelMap) throws RawDataUnavailableException {
 		RawDataUnit firstUnit = getFirstUnit();
 		if (firstUnit == null) {
-			throw new RawDataUnavailableException(this, RawDataUnavailableException.DATA_NOT_ARRIVED, startMillis, (int) durationMillis);
+			throw new RawDataUnavailableException(this, RawDataUnavailableException.DATA_NOT_ARRIVED, 0,0, startMillis, (int) durationMillis);
 		}
 		long firstMillis = firstUnit.getTimeMilliseconds();
 		long firstSamples = firstUnit.getStartSample();
 		RawDataUnit lastUnit = getLastUnit();
 		long lastMillis = lastUnit.getEndTimeInMilliseconds();
+		long lastSample = lastUnit.getStartSample()+lastUnit.getSampleDuration();
 		
 		
 		long firstAvailableMillis = Math.max(firstMillis, startMillis);
@@ -272,7 +273,8 @@ public class PamRawDataBlock extends AcousticDataBlock<RawDataUnit> {
 		double[][] data = getSamplesForMillis(firstAvailableMillis, lastAvailableMillis-firstAvailableMillis, channelMap);
 		if (data == null) {
 			// this shouldn't happen. If an exception wasn't thrown from getSamples... then data should no tb enull
-			throw new RawDataUnavailableException(this, RawDataUnavailableException.DATA_NOT_ARRIVED, startMillis, (int) durationMillis);
+			throw new RawDataUnavailableException(this, RawDataUnavailableException.DATA_NOT_ARRIVED,
+					firstSamples, lastSample,	startMillis, (int) durationMillis);
 		}
 		RawDataUnit[] dataUnits = new RawDataUnit[data.length];
 		for (int i = 0; i < data.length; i++) {
@@ -298,7 +300,7 @@ public class PamRawDataBlock extends AcousticDataBlock<RawDataUnit> {
 	synchronized public double[][] getSamplesForMillis(long startMillis, long durationMillis, int channelMap) throws RawDataUnavailableException {
 		RawDataUnit firstUnit = getFirstUnit();
 		if (firstUnit == null) {
-			throw new RawDataUnavailableException(this, RawDataUnavailableException.DATA_NOT_ARRIVED, startMillis, (int) durationMillis);
+			throw new RawDataUnavailableException(this, RawDataUnavailableException.DATA_NOT_ARRIVED, 0, 0, startMillis, (int) durationMillis);
 		}
 		long firstMillis = firstUnit.getTimeMilliseconds();
 		long firstSamples = firstUnit.getStartSample();
@@ -317,23 +319,28 @@ public class PamRawDataBlock extends AcousticDataBlock<RawDataUnit> {
 		// run  a few tests ...
 		int chanOverlap = channelMap & getChannelMap();
 		if (chanOverlap != channelMap) {
-			throw new RawDataUnavailableException(this, RawDataUnavailableException.INVALID_CHANNEL_LIST, startSample, duration);
+			throw new RawDataUnavailableException(this, RawDataUnavailableException.INVALID_CHANNEL_LIST, 0,0,startSample, duration);
 		}
 		if (duration < 0) {
-			throw new RawDataUnavailableException(this, RawDataUnavailableException.NEGATIVE_DURATION, startSample, duration);
+			throw new RawDataUnavailableException(this, RawDataUnavailableException.NEGATIVE_DURATION,0,0, startSample, duration);
 		}
 		
 		RawDataUnit dataUnit = getFirstUnit();
 		if (dataUnit == null) {
 			return null;
 		}
-		if (dataUnit.getStartSample() > startSample) {
+		RawDataUnit lastUnit = getLastUnit();
+		long firstSample = dataUnit.getStartSample();
+		long lastSample = lastUnit.getStartSample()+lastUnit.getSampleDuration();
+		if (firstSample > startSample) {
 //			System.out.println("Earliest start sample : " + dataUnit.getStartSample());
-			throw new RawDataUnavailableException(this, RawDataUnavailableException.DATA_ALREADY_DISCARDED, startSample, duration);
+			throw new RawDataUnavailableException(this, RawDataUnavailableException.DATA_ALREADY_DISCARDED, 
+					firstSample, lastSample, startSample, duration);
 		}
 		dataUnit = getLastUnit();
 		if (hasLastSample(dataUnit, startSample+duration, channelMap) == false)  {
-			throw new RawDataUnavailableException(this, RawDataUnavailableException.DATA_NOT_ARRIVED, startSample, duration);
+			throw new RawDataUnavailableException(this, RawDataUnavailableException.DATA_NOT_ARRIVED,
+					firstSample, lastSample, startSample, duration);
 		}
 		
 		int nChan = PamUtils.getNumChannels(channelMap);

src/PamguardMVC/RawDataUnavailableException.java

@@ -21,6 +21,10 @@ public class RawDataUnavailableException extends Exception {
 	private long startSample;
 
 	private int duration;
+
+	private long availableStart;
+
+	private long availableEnd;
 	/**
 	 * @return the dataCause
 	 */
@@ -34,10 +38,12 @@ public class RawDataUnavailableException extends Exception {
 	 * @param startSample 
 	 * @param cause
 	 */
-	public RawDataUnavailableException(PamRawDataBlock rawDataBlock, int dataCause, long startSample, int duration) {
+	public RawDataUnavailableException(PamRawDataBlock rawDataBlock, int dataCause, long availStart, long availEnd, long startSample, int duration) {
 		super();
 		this.rawDataBlock = rawDataBlock;
 		this.dataCause = dataCause;
+		this.availableStart = availStart;
+		this.availableEnd = availEnd;
 		this.startSample = startSample;
 		this.duration = duration;
 	}
@@ -55,8 +61,8 @@ public class RawDataUnavailableException extends Exception {
 	public String getMessage() {
 		switch (dataCause) {
 		case DATA_ALREADY_DISCARDED:
-			return String.format("Samples %d length %d requested from %s have already been discarded", startSample, duration, 
-					rawDataBlock.getDataName());
+			return String.format("Samples %d length %d requested from %s have already been discarded. %d to %d available", startSample, duration, 
+					rawDataBlock.getDataName(), availableStart, availableEnd);
 		case DATA_NOT_ARRIVED:
 			return String.format("Samples %d length %d  requested from %s have not yet arrived", 
 					startSample, duration, rawDataBlock.getDataName());
@@ -71,5 +77,19 @@ public class RawDataUnavailableException extends Exception {
 		return super.getMessage();
 	}
 
+	/**
+	 * @return the availableStart
+	 */
+	public long getAvailableStart() {
+		return availableStart;
+	}
+
+	/**
+	 * @return the availableEnd
+	 */
+	public long getAvailableEnd() {
+		return availableEnd;
+	}
+	
 
 }

src/PamguardMVC/ThreadedObserver.java

@@ -8,6 +8,7 @@ import Acquisition.AcquisitionControl;
 import Acquisition.AcquisitionProcess;
 import Acquisition.DaqSystem;
 import PamController.PamController;
+import PamModel.PamModel;
 import PamUtils.PamCalendar;
 import PamguardMVC.debug.Debug;
 
@@ -130,6 +131,7 @@ public class ThreadedObserver implements PamObserver {
 				}
 			}
 		}
+		h += PamModel.getPamModel().getPamModelSettings().getThreadingJitterMillis()*2;
 
 		return h;
 	}

src/binaryFileStorage/BinaryStore.java

@@ -228,6 +228,7 @@ PamSettingsSource, DataOutputStore {
 		super.pamToStart();
 		prepareStores();
 		openStores();
+		binaryStoreProcess.checkFileTimer();
 	}
 
 	@Override
@@ -245,9 +246,9 @@ PamSettingsSource, DataOutputStore {
 	 * Called from the process to close and reopen each datastream in 
 	 * a new file. Probably gets called about once an hour on the hour. 
 	 */
-	protected void reOpenStores(int endReason) {
+	protected synchronized void reOpenStores(int endReason, long newFileTime) {
 
-		long dataTime = PamCalendar.getTimeInMillis();
+		long dataTime = newFileTime;//PamCalendar.getTimeInMillis();
 		long analTime = System.currentTimeMillis();
 		BinaryOutputStream dataStream;
 		for (int i = 0; i < storageStreams.size(); i++) {
@@ -536,7 +537,7 @@ PamSettingsSource, DataOutputStore {
 				 */
 				if (immediateChanges) {
 					if (storesOpen) {
-						reOpenStores(BinaryFooter.END_UNKNOWN);
+						reOpenStores(BinaryFooter.END_UNKNOWN, PamCalendar.getTimeInMillis());
 					}
 				}
 				
@@ -2601,5 +2602,11 @@ PamSettingsSource, DataOutputStore {
 	public DataIntegrityChecker getInegrityChecker() {
 		return new BinaryIntegrityChecker(this);
 	}
+	/**
+	 * @return the binaryStoreProcess
+	 */
+	public BinaryStoreProcess getBinaryStoreProcess() {
+		return binaryStoreProcess;
+	}
 	
 }

src/binaryFileStorage/BinaryStoreProcess.java

@@ -16,6 +16,8 @@ public class BinaryStoreProcess extends PamProcess {
 
 	private Timer timer;
 
+	private Object timerSynch = new Object();
+
 	public BinaryStoreProcess(BinaryStore binaryStore) {
 		super(binaryStore, null);
 		this.binaryStore = binaryStore;
@@ -27,17 +29,21 @@ public class BinaryStoreProcess extends PamProcess {
 		return "Binary store file control";
 	}
 
-	public synchronized void checkFileTime() {
+	public synchronized void checkFileTime(long masterClockTime) {
+		//		if (binaryStore.binaryStoreSettings.autoNewFiles && 
+		//				 PamCalendar.getTimeInMillis() >= nextFileTime) {
+		//			startNewFiles();
+		//		}
 		if (binaryStore.binaryStoreSettings.autoNewFiles && 
-				 PamCalendar.getTimeInMillis() >= nextFileTime) {
-			startNewFiles();
+				masterClockTime >= nextFileTime) {
+			startNewFiles(masterClockTime);
 		}
 
 	}
 
-	private synchronized void startNewFiles() {
+	private synchronized void startNewFiles(long masterClockTime) {
 		nextFileTime += binaryStore.binaryStoreSettings.fileSeconds * 1000;
-		binaryStore.reOpenStores(BinaryFooter.END_FILETOOLONG);
+		binaryStore.reOpenStores(BinaryFooter.END_FILETOOLONG, masterClockTime);
 	}
 
 
@@ -47,23 +53,54 @@ public class BinaryStoreProcess extends PamProcess {
 		long round = binaryStore.binaryStoreSettings.fileSeconds * 1000;
 		nextFileTime = (startTime/round) * round + round;
 		//		System.out.println("Next file start at " + PamCalendar.formatDateTime(nextFileTime));
+	}
+
+	public void checkFileTimer() {
+		boolean needTimer = PamCalendar.isSoundFile() == false;
+		if (needTimer) {
+			startTimer();
+		}
+		else {
+			stopTimer();
+		}
+	}
+	
+	private void startTimer() {
+		synchronized (timerSynch) {
+			if (timer == null) {
 				timer = new Timer();
 				timer.schedule(new FileTimerTask(), 1000, 1000);
-		
 			}
+		}
+	}
+	
+	private void stopTimer() {
+		synchronized (timerSynch) {
+			if (timer != null) {
+				timer.cancel();
+				timer = null;
+			}
+		}
+	}
+
+
+
+	//	@Override
+	//	public void masterClockUpdate(long timeMilliseconds, long sampleNumber) {
+	//		super.masterClockUpdate(timeMilliseconds, sampleNumber);
+	//		checkFileTime(timeMilliseconds);
+	//	}
 
 	class FileTimerTask extends TimerTask {
 		@Override
 		public void run() {
-			checkFileTime();
+			checkFileTime(PamCalendar.getTimeInMillis());
 		}
 	}
 
 	@Override
 	public void pamStop() {
-		if (timer != null) {
-			timer.cancel();
-		}
+		stopTimer();
 	}
 
 }

src/clipgenerator/ClipProcess.java

@@ -140,10 +140,13 @@ public class ClipProcess extends SpectrogramMarkProcess {
 				clipErr = clipRequest.clipBlockProcess.processClipRequest(clipRequest);
 				switch (clipErr) {
 				case 0: // no error - clip should have been created. 
+					li.remove();
+					break;
 				case RawDataUnavailableException.DATA_ALREADY_DISCARDED:
 				case RawDataUnavailableException.INVALID_CHANNEL_LIST:
 //					System.out.println("Clip error : " + clipErr);
 					li.remove();
+					break;
 				case RawDataUnavailableException.DATA_NOT_ARRIVED:
 					continue; // hopefully, will get this next time !
 				}
@@ -230,6 +233,17 @@ public class ClipProcess extends SpectrogramMarkProcess {
 			}
 			minH = Math.max(minH, clipBlockProcesses[i].getRequiredDataHistory(o, arg));
 		}
+		
+		ClipRequest firstClip = null;
+		synchronized(clipRequestSynch) {
+			if (clipRequestQueue.size() > 0) {
+				firstClip = clipRequestQueue.get(0);
+			}
+		}
+		if (firstClip != null) {
+			minH += firstClip.dataUnit.getDurationInMilliseconds();
+		}
+		
 		minH += Math.max(3000, 192000/(long)getSampleRate());
 		if (specMouseDown) {
 			minH = Math.max(minH, masterClockTime-specMouseDowntime);
@@ -453,8 +467,7 @@ public class ClipProcess extends SpectrogramMarkProcess {
 			this.dataBlock = dataBlock;
 			this.clipGenSetting = clipGenSetting;
 			clipBudgetMaker = new StandardClipBudgetMaker(this);
-			dataBlock.addObserver(this, true);
-			
+			dataBlock.addObserver(this, false);
 
 			if (rawDataBlock != null) {
 				int chanMap = decideChannelMap(rawDataBlock.getChannelMap());
@@ -499,6 +512,7 @@ public class ClipProcess extends SpectrogramMarkProcess {
 				rawData = rawDataBlock.getSamples(rawStart, (int) (rawEnd-rawStart), channelMap);
 			}
 			catch (RawDataUnavailableException e) {
+				System.out.println(e.getMessage());
 				return e.getDataCause();
 			}
 			if (rawData==null) {
@@ -583,9 +597,15 @@ public class ClipProcess extends SpectrogramMarkProcess {
 		public PamObserver getObserverObject() {
 			return clipProcess.getObserverObject();
 		}
+	
 		@Override
 		public long getRequiredDataHistory(PamObservable o, Object arg) {
-			return (long) ((clipGenSetting.preSeconds+clipGenSetting.postSeconds) * 1000.);
+			long h = (long) ((clipGenSetting.preSeconds+clipGenSetting.postSeconds) * 1000.);
+//			if (dataBlock != null) {
+			// can't do this since dataBlock is observing this, so will wrap. 
+//				h += dataBlock.getRequiredHistory();
+//			}
+			return h;
 		}
 		
 		

src/tethys/dbxml/DBXMLConnect.java

@@ -150,8 +150,7 @@ public class DBXMLConnect {
 	 */
 	public boolean postAndLog(Object nilusObject, String documentName) throws TethysException
 	{
-		boolean ok = NilusChecker.warnEmptyFields(tethysControl.getGuiFrame(), nilusObject);
-
+//		boolean ok = NilusChecker.warnEmptyFields(tethysControl.getGuiFrame(), nilusObject);
 
 		TethysException e = null;
 		boolean success = false;