Merge pull request #73 from PAMGuard/main

merge from main

pom.xml

@@ -5,7 +5,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.pamguard</groupId>
   <artifactId>Pamguard</artifactId>
-  <version>2.02.10b</version>
+  <version>2.02.10ad</version>
   <name>Pamguard Java12+</name>
   <description>Pamguard for Java 12+, using Maven to control dependcies</description>
 	<url>www.pamguard.org</url>

src/Array/ArrayManager.java

@@ -25,6 +25,7 @@ import Array.sensors.swing.ArraySensorPanelProvider;
 import GPS.GPSControl;
 import GPS.GPSDataBlock;
 import GPS.GpsData;
+import GPS.GpsDataUnit;
 import PamController.PamControlledUnit;
 import PamController.PamControlledUnitGUI;
 import PamController.PamControlledUnitSettings;
@@ -32,6 +33,7 @@ import PamController.PamController;
 import PamController.PamGUIManager;
 import PamController.PamSettingManager;
 import PamController.PamSettings;
+import PamController.masterReference.MasterReferencePoint;
 import PamController.positionreference.PositionReference;
 import PamModel.PamModuleInfo;
 import PamUtils.PamUtils;
@@ -993,6 +995,7 @@ public class ArrayManager extends PamControlledUnit implements PamSettings, PamO
 	 * @return geometry data. 
 	 */
 	public SnapshotGeometry getSnapshotGeometry(int hydrophoneMap, long timeMillis) {
+				
 		PamArray currentArray = getCurrentArray();
 		if (currentArray == null) {
 			return null;
@@ -1080,14 +1083,45 @@ public class ArrayManager extends PamControlledUnit implements PamSettings, PamO
 			}
 		}
 
-		if (nGood > 0) for (int p = 0; p < 3; p++) {
-			centre[p] /= nGood;
+		if (nGood > 0) { 
+			for (int p = 0; p < 3; p++) {
+				centre[p] /= nGood;
+			}
+			return new SnapshotGeometry(currentArray, timeMillis, streamerList, hydrophoneList, firstStreamerPos,
+					new PamVector(centre), geometry, streamerError, hydrophoneError);
 		}
-		
-		return new SnapshotGeometry(currentArray, timeMillis, streamerList, hydrophoneList, firstStreamerPos,
-				new PamVector(centre), geometry, streamerError, hydrophoneError);
+		else {
+			return getMasterReferenceGeometry(timeMillis);
+		}	
 		
 	}
+	
+	/**
+	 * Create a snapshot geometry from the master reference position, which will either be the GPS data, or 
+	 * the centre point of the array. Worst case is that it ends up as 0,0,0
+	 * @param timeMillis
+	 * @return
+	 */
+	private SnapshotGeometry getMasterReferenceGeometry(long timeMillis) {
+		GPSControl gpsControl = GPSControl.getGpsControl();
+		GpsData referencePoint = null;
+		if (gpsControl != null) {
+			GpsDataUnit shipPos = gpsControl.getShipPosition(timeMillis, true);
+			if (shipPos != null) {
+				referencePoint = shipPos.getGpsData();
+			}
+		}
+		if (referencePoint == null) {
+			// running out of options, so fall back to the master reference point, interpolated (probably has zero speeed)
+			referencePoint = new GpsData(MasterReferencePoint.getFixTime(), MasterReferencePoint.getLatLong()).getPredictedGPSData(timeMillis);
+		}
+		if (referencePoint == null) {
+			return null;
+		}
+		SnapshotGeometry snapgeom = new SnapshotGeometry(getCurrentArray(), timeMillis, null, null, referencePoint, new PamVector(0,0,0), null, null, null);
+		return snapgeom;
+	}
+
 //	
 //	public SnapshotGeometry getSubDetectionGeometry(PamDataUnit superDataUnit) {
 //		/**

src/GPS/GPSControl.java

@@ -4,6 +4,7 @@ import java.awt.Frame;
 import java.awt.event.ActionEvent;
 import java.awt.event.ActionListener;
 import java.io.Serializable;
+import java.util.ListIterator;
 
 import javax.swing.JMenu;
 import javax.swing.JMenuItem;
@@ -22,6 +23,7 @@ import PamController.PamSettings;
 import PamController.positionreference.PositionReference;
 import PamUtils.PamCalendar;
 import PamView.dialog.warn.WarnOnce;
+import PamguardMVC.PamDataBlock;
 
 public class GPSControl extends PamControlledUnit implements PamSettings, PositionReference {
 
@@ -242,9 +244,68 @@ public class GPSControl extends PamControlledUnit implements PamSettings, Positi
 		return super.removeUnit();
 	}
 	
+	/**
+	 * Gets the closest position based on time. No interpolation
+	 * @param timeMilliseconds
+	 * @return closest ship position based on time. 
+	 */
 	public GpsDataUnit getShipPosition(long timeMilliseconds) {
 		return getGpsDataBlock().getClosestUnitMillis(timeMilliseconds);
 	}
+
+	/**
+	 * Gets the closest position based on time. 
+	 * @param timeMilliseconds time in milliseconds
+	 * @param interpolate interpolate between the point before and the one after. 
+	 * @return interpolated gps position. 
+	 */
+	public GpsDataUnit getShipPosition(long timeMilliseconds, boolean interpolate) {
+ 		if (interpolate == false) {
+			return getGpsDataBlock().getClosestUnitMillis(timeMilliseconds);
+		}
+		// otherwise try to fine a point either side and weighted mean them or extrapolate. 
+		GpsDataUnit pointBefore = null, pointAfter = null;
+		synchronized (getGpsDataBlock().getSynchLock()) {
+			ListIterator<GpsDataUnit> iter = getGpsDataBlock().getListIterator(timeMilliseconds, 0, PamDataBlock.MATCH_BEFORE, PamDataBlock.POSITION_BEFORE);
+			if (iter.hasNext()) {
+				pointBefore = iter.next();
+			}
+			if (iter.hasNext()) {
+				pointAfter = iter.next();
+			}
+		}
+		if (pointBefore == null && pointAfter == null) {
+			return null;
+		}
+		if (pointBefore != null & pointAfter != null) {
+			// interpolate since we have them both. 
+			double tB = timeMilliseconds-pointBefore.getTimeMilliseconds();
+			double tA = pointAfter.getTimeMilliseconds()-timeMilliseconds;
+			double wB = tA/(tB+tA); // weight for data before
+			double wA = tB/(tB+tA); // wedight for data after
+			double newLat = pointBefore.getGpsData().getLatitude()*wB + pointAfter.getGpsData().getLatitude()*wA;
+			double newLon = pointBefore.getGpsData().getLongitude()*wB + pointAfter.getGpsData().getLongitude()*wA;
+			// interpolating heading is more faff:
+			double hB = pointBefore.getGpsData().getHeading();
+			double hA = pointAfter.getGpsData().getHeading();
+			double hX = Math.sin(Math.toRadians(hB))*wB + Math.sin(Math.toRadians(hB))*wB;
+			double hY = Math.cos(Math.toRadians(hB))*wB + Math.cos(Math.toRadians(hB))*wB;
+			double newHead = Math.toDegrees(Math.atan2(hX, hY));
+			GpsData newData = new GpsData(newLat, newLon, 0, timeMilliseconds);
+			newData.setTrueHeading(newHead);
+			return new GpsDataUnit(timeMilliseconds, newData);
+		}
+		else if (pointBefore != null) {
+			// relatively common if we enter data just after a position was received. 
+			GpsData newPos = pointBefore.getGpsData().getPredictedGPSData(timeMilliseconds);
+			return new GpsDataUnit(timeMilliseconds, newPos);
+		}
+		else if (pointAfter != null) {
+			GpsData newPos = pointAfter.getGpsData().getPredictedGPSData(timeMilliseconds);
+			return new GpsDataUnit(timeMilliseconds, newPos);
+		}
+		return null; // should never happen since one of the above 4 clauses must be met. 
+	}
 	/**
 	 * Do we want this string ? It will be either RMC or GGA and may want wildcarding
 	 * @param stringId
@@ -322,6 +383,7 @@ public class GPSControl extends PamControlledUnit implements PamSettings, Positi
 	public GPSDataMatcher getGpsDataMatcher() {
 		return gpsDataMatcher;
 	}
+	
 	@Override
 	public GpsData getReferencePosition(long timeMillis) {
 		GpsDataUnit gpDU = getShipPosition(timeMillis);

src/PamController/PamConfiguration.java

@@ -184,6 +184,23 @@ public class PamConfiguration {
 				return dataBlock;
 			}
 		}
+		return tryShortName(blockType, name);
+	}
+	
+	/**
+	 * For backwards compatibility, some blocks may still use the short name.
+	 * @param blockType
+	 * @param name
+	 * @return
+	 */
+	private PamDataBlock tryShortName(Class blockType, String name) {
+		if (name == null) return null;
+		ArrayList<PamDataBlock> blocks = getDataBlocks(blockType, true);
+		for (PamDataBlock dataBlock:blocks) {
+			if (name.equals(dataBlock.getDataName())) { // check for a long name match first 
+				return dataBlock;
+			}
+		}
 		return null;
 	}
 

src/PamController/PamController.java

@@ -196,7 +196,7 @@ public class PamController implements PamControllerInterface, PamSettings {
 
 	private Timer diagnosticTimer;
 	
-	private boolean debugDumpBufferAtRestart = false;
+	private boolean debugDumpBufferAtRestart = true;
 
 	private NetworkController networkController;
 	private int nNetPrepared;
@@ -1434,7 +1434,7 @@ public class PamController implements PamControllerInterface, PamSettings {
 				pamControlledUnits.get(iU).flushDataBlockBuffers(2000);
 			}
 		}
-		dumpBufferStatus("In pamStopped, now idle", true);
+		dumpBufferStatus("In pamStopped, now idle", false);
 
 		// wait here until the status has changed to Pam_Idle, so that we know
 		// that we've really finished processing all data

src/PamController/PamguardVersionInfo.java

@@ -31,12 +31,12 @@ public class PamguardVersionInfo {
 	 * Version number, major version.minorversion.sub-release.
 	 * Note: can't go higher than sub-release 'f'
 	 */
-	static public final String version = "2.02.10b";
+	static public final String version = "2.02.10ad";
 	
 	/**
 	 * Release date
 	 */
-	static public final String date = "2 March 2024";
+	static public final String date = "13 March 2024";
 	
 //	/**
 //	 * Release type - Beta or Core

src/PamguardMVC/PamDataBlock.java

@@ -4296,6 +4296,9 @@ public class PamDataBlock<Tunit extends PamDataUnit> extends PamObservable {
 	 * @param sayEmpties dump info even if a buffer is empty (otherwise, only ones that have stuff still)
 	 */
 	public void dumpBufferStatus(String message, boolean sayEmpties) {
+		if (sayEmpties || unitsAdded > 0) {
+			System.out.printf("Datablock %s: Added %d data units\n", getLongDataName(), unitsAdded);
+		}
 		int nObs = countObservers();
 		for (int i = 0; i < nObs; i++) {
 			PamObserver obs = getPamObserver(i);

src/PamguardMVC/PamDataUnit.java

@@ -564,7 +564,8 @@ abstract public class PamDataUnit<T extends PamDataUnit, U extends PamDataUnit>
 	
 	/**
 	 * Get the latlong of the mean hydrophone position at the time of 
-	 * this detection. 
+	 * this detection. If the data unit has a channel bitmap of zero, then 
+	 * get the GPS position of the vessel at that time. 
 	 * @param recalculate
 	 * @return Lat long of detection origin (usually the position of the reference hydrophone at time of detection)
 	 */

src/PamguardMVC/PamProcess.java

@@ -126,6 +126,8 @@ abstract public class PamProcess implements PamObserver, ProcessAnnotator {
 	private long cpuUsage;
 	private long lastCPUCheckTime = System.currentTimeMillis();
 	private double cpuPercent;
+	private double totalCPU, peakCPU;
+	private int nCPU;
 	
 	/**
 	 * Flag for the process to say whether or not it's primary data connection
@@ -140,6 +142,11 @@ abstract public class PamProcess implements PamObserver, ProcessAnnotator {
 	 * Last received data unit - used for working out timing offsets. 
 	 */
 	private PamDataUnit lastAcousticDataunit;
+
+	/*
+	 * diagnostic count of all data processed
+	 */
+	private int nDataProcessed;
 	// some flags and variables needed to deal with conversion from
 	// milliseconds to samples and back
 //	private boolean acousticDataSource = false;
@@ -742,6 +749,7 @@ abstract public class PamProcess implements PamObserver, ProcessAnnotator {
 		}
 //		long cpuStart = SystemTiming.getProcessCPUTime();
 		long threadId = Thread.currentThread().getId();
+		nDataProcessed++;
 		long cpuStart = tmxb.getThreadCpuTime(threadId);
 			newData(o, arg);
 			if (processCheck != null) {
@@ -762,6 +770,12 @@ abstract public class PamProcess implements PamObserver, ProcessAnnotator {
 			 * get percent. Total is -9+3+2 = /!0^4 !  
 			 */
 			cpuPercent = (double) cpuUsage / (now - lastCPUCheckTime) / 10000.;
+			// these next two allow us to take an average cpu. 
+			totalCPU += cpuPercent;
+			nCPU++;
+			// and hte max cpu (may always go to 100 at end ?)
+			peakCPU = Math.max(cpuPercent, peakCPU);
+			
 			lastCPUCheckTime = now;
 			cpuUsage = 0;
 		}
@@ -1073,6 +1087,9 @@ abstract public class PamProcess implements PamObserver, ProcessAnnotator {
 	 * @param sayEmpties include info even if a buffer is empty. 
 	 */
 	public void dumpBufferStatus(String message, boolean sayEmpties) {
+		if (sayEmpties || nDataProcessed > 0) {
+			System.out.printf("Process %s: ran %d datas, peak CPU %3.1f%%, mean CPU %3.1f%%\n", this.getProcessName(), nDataProcessed, peakCPU, totalCPU/nCPU);
+		}
 		ArrayList<PamDataBlock> outputs = getOutputDataBlocks();
 		try {
 			for (PamDataBlock output : outputs) {

src/help/JavaHelpSearch/SCHEMA

@@ -1,2 +1,2 @@
 JavaSearch 1.0
-TMAP bs=2048 rt=1 fl=-1 id1=6882 id2=1
+TMAP bs=2048 rt=1 fl=-1 id1=6905 id2=1

src/help/Map.jhm

@@ -686,6 +686,8 @@
 
 <mapID target="detectors.clickDetectorHelp.docs.ClickDetector_clickSidePanel" url="detectors/clickDetectorHelp/docs/ClickDetector_clickSidePanel.html"/>
 
+<mapID target="classifiers.matchedtemplate.mathchedtemplate" url="classifiers/matchedtemplate/mathchedtemplate.html"/>
+
 <mapID target="localisation.detectiongroup.docs.dglocaliser" url="localisation/detectiongroup/docs/dglocaliser.html"/>
 
 <mapID target="visual_methods.loggerFormsHelp.docs.SHORT" url="visual_methods/loggerFormsHelp/docs/SHORT.html"/>

src/help/PAMGUARDHelpProject.xml

@@ -6,7 +6,7 @@
   <project>
 PAMGUARD
   </project>
-  <projectdir>C:\Users\dg50\source\repos\PAMGuardDG_2\src\help</projectdir>
+  <projectdir>C:\Users\dg50\source\repos\PAMGuardPAMGuard\src\help</projectdir>
   <startpage>
 index.html
   </startpage>

src/help/PAMGUARDTOC.xml

@@ -557,6 +557,10 @@
    <tocitem text="Common Mistakes and Bugs " target="classifiers.rawDeepLearningHelp.docs.rawDeepLearning_Bugs" image="topic"/>
 
   </tocitem>
+  <tocitem text="Matched Template Classifier ">
+   <tocitem text="Matched Template Classifier " target="classifiers.matchedtemplate.mathchedtemplate" image="topic"/>
+
+  </tocitem>
  </tocitem>
  <tocitem text="Localisation ">
   <tocitem text="Overview " target="localisation.docs.localisation_overview" image="topic"/>

src/help/classifiers/matchedtemplate/mathchedtemplate.html

@@ -0,0 +1,71 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html><head>
+<link href="../../pamHelpStylesheet.css" type="text/css" rel="STYLESHEET"><title>ROCCA Classifier Overview</title>
+
+</head>
+<body>
+<h1 id="matched-click-classifier">Matched click classifier</h1>
+<h2 id="overview">Overview</h2>
+<p>The matched click classifier is an alternative to the in built click classifier in the click detection module which uses two click templates (target and reject) to classify individual clicks detections from the click detection module. The idea behind this classifier is to more accurately classify rarer clicks when there </p>
+<h2 id="how-it-works">How it works</h2>
+<p>The classifier is based on a matched filter i.e. a candidate click detection is compared to a template and it&#39;s maximum correlation value with the template is then used for classification. Each click is compared to both a <em>match</em> and a <em>reject</em> template. If the difference between the correlation match to the reject template exceeds a certain threshold, then the click is classified. There can be multiple combinations of match to reject templates allowing the matched classifier to deal with different types of clicks or even clicks from multiple species.</p>
+<h2 id="configuring-the-matched-click-classifier">Configuring the matched click classifier</h2>
+<p>The matched click classifier settings are accessed via <strong>Settings-&gt; Matched click classifier</strong>_. The settings are split into the three sections, general settings, click waveform and click templates. </p>
+<p align="center">
+  <img width="950" height="520" src = "resources/matched_click_dialog_summary.png">
+</p>
+
+<p><em>The settings pane of the matched click classifier</em></p>
+<h3 id="general-settings">General Settings</h3>
+<p>The general settings allow for channel options, species ID and the default colours for classified clicks to be to be set. 
+_
+Channel Options _allows users to define whether a single click from one channel, all clicks or an averaged click should be used to be used for classification in multi-channel situations. if there is only one channel then this makes no difference. </p>
+<p><em>Click Type</em> sets the number that defines the species ID. Make sure this is not the same as any of the species IDs in the default click classifier (this is why the default is set so high). </p>
+<p><em>Symbol</em> and <em>Fill</em> define the default colours clicks which have been classified by the matched click classifier should be plotted on displays. </p>
+<h3 id="click-waveform-settings">Click Waveform Settings</h3>
+<p>Before a click is classified it undergoes some pre-conditioning by the matched click classifier. </p>
+<p><em>Restrict parameter extraction to XX samples</em> sets the maximum length of the waveform to the classifier. If this is selected, then center of the click is located and samples trimmed around the center. In some use cases, for example SoundTrap detections, which may be 10,000 samples long, setting a lower number of maximum samples can greatly increase processing speed and improve the accuracy of results. Note that if the number of samples is set too low (e.g. well below the click template length) then the correaltion values will not make much sense.</p>
+<p><em>Peak threshold</em> and <em>Smoothing</em> are both parameters used to find the click center so that a click can be trimmed to the maximum number of samples. The click length is measured by calculating the waveform envelope using a Hilbert Transform. The envelope is smoothed using a moving average filter (the <em>Smoothing</em> parameter defines the size of the averaging window). The click is trimmed as follows. First the peak of the waveform envelope is found. The length of the click is defined as the point at which the click falls <em>Peak threshold</em> dB below the peak. The center of the click is then the middle of this snippet. The click is then trimmed from the center of the click. </p>
+<p><em>Amplitude normalisation</em> If there is a very loud click compared to a template it&#39;s correlation score will be different to that of a very quiet click of with exactly the same waveform. It is therefore a good idea to normalise the waveform before it is compared the match click classifier. The types of normalisation are </p>
+<ul>
+<li><em>norm</em> - this is the default - the click is divided by it&#39;s RMS amplitude. </li>
+<li><em>peak to peak</em> this can be useful for some types of shorter click e.g. dolphins - the click is divided by it&#39;s peak to peak amplitude.</li>
+<li><em>none</em> no normalisation (not recommended).</li>
+</ul>
+<h3 id="template-settings">Template settings</h3>
+<p>The matched click classifier has two templates, a match and a reject. The match template should match a stereotypical click of the target species and the reject template can either be a click from a confounding species (e.g. dolphin) or blank. Selecting the drop-down menu from the <em>Import</em> button allows for a selection of some default templates. Custom templates can be imported by selecting the <em>Import</em> button. Custom templates can either be a .csv. or .mat file using the following format. </p>
+<p><strong>.csv</strong> - the first <em>row</em> are the waveform measurements from -1 to 1 (make sure you save with a sufficient number of decimal points). The second row and first column is the sample rate in sample per second.</p>
+<p><strong>.mat</strong> - a .mat file that contains two variables named <em>sR</em> and <em>waveform</em>. <em>sR</em> is the sample rate in samples per second and <em>waveform</em> is a 1D array containing the waveform for the template form -1 to 1. Arrays of click structures imported using the PAMGuard to MATLAB library can also be used. The first channel of the first click in a list of clicks structures will be imported as the template. The click structure should be names <em>clicks</em> and the sample rate should be saved as a seperate variable named <em>sR</em> in sampes per second.</p>
+<p>The match and reject templates are plotted to provide a user with some visualisation of the classifier settings - the drop down menu allows the user to select different ways to plot the templates and is purely for visualisation purposes so makes no difference to classifier settings. </p>
+<p><em>threshold</em> is the threshold at which a click is classified. If the difference between the match and reject templates is above the <em>threshold</em> value then the 
+click is classified. </p>
+<p>The + button can be used to add more tabs. Each tab contains a click/reject template pair and unique threshold setting. A click is classified if at least one of the match/reject templates passes it&#39;s set threshold. </p>
+<h2 id="visualising-results">Visualising Results</h2>
+<p>The matched click classifier changes the species type flag of a click if at least one of the classifiers passes threshold. This means clicks can be visualised in the click bearing time display or the time base display. The classifier also saves the correlation values for each match/reject template pair which can be accessed in MATLAB/R or through the PAMGuard GUI. </p>
+<h3 id="click-bearing-time-display">Click bearing time display</h3>
+<p>Matched clicks can be viewed in the bearing time display. If a click passes the threshold of one match/reject pair then the click symbol (defined in general settings) is shown in the bearing time display if <em>Colour by Matched Template Classifier</em> is selected in the right click menu. The correlation values are shown by hovering the mouse over a click to bring up the info tool tip. </p>
+<p align="center">
+  <img width="950" height="550" src = "resources/matched_click_bt_display.png">
+</p>
+
+<p><em>Screenshot of clicks classified from the matched click classifier showing matched clicks (coloured pink)</em></p>
+<h3 id="time-base-display">Time base display</h3>
+<p>The time base display FX can show clicks classified by the matched clicks classifier. <em>Colour by Matched Template Classifier</em> must be selecting in the right settings window. </p>
+<p align="center">
+  <img width="950" height="520" src = "resources/matched_click_tdisplay_example.png">
+</p>
+
+<p><em>Screenshot of Time Base display FX showing clicks classified by the match click classifier (coloured pink). The correlation values can be found in the meta data section of the detection pop up menu (highlighted)</em></p>
+<p>Note that the time base display allows users to export clicks to be used as templates. Using the advanced pop up menu right click on a click detection and select the MATLAB icon. A .mat file of the selected click or clicks will be saved to your root user folder in a folder called <em>PAMGuard Manual Export</em>.  This .mat file can be opened by the matched click classifier - the first channel of the first click in the list will be imported as a template. </p>
+<p align="center">
+  <img width="510" height="300" src = "resources/exporting_mat_clicks.png">
+</p>
+
+
+<h3 id="extracting-correlation-values-using-matlab-r">Extracting correlation values using MATLAB/R</h3>
+<p>Clicks are loaded from binary files using the MATLAB/R function </p>
+<pre><code><span class="hljs-attribute">clicks</span>  = loadPamguardBinaryFile(<span class="hljs-string">"/path/to/pamguardfile.pgdf"</span>)<span class="hljs-comment">;</span>
+</code></pre><p>where clicks is a list of MATLAB/R structures containing the data for each click in the file. </p>
+<p>The matched click classifier template threshold, match value and reject value are then accessed from each click using</p>
+<pre><code><span class="hljs-attr">matchedtemplatevals</span> = clicks(<span class="hljs-number">1</span>).annotations. mclassification
+</code></pre><p><code>matachedtemplatevals</code> is a list of where each row is the threshold value, match value and reject value for each match/reject template pair. </p>

src/serialComms/SerialPortPanel.java

@@ -236,9 +236,12 @@ public class SerialPortPanel extends Object {
 		}
 	}
 	
-
 	public String getPort() {
 		if (portList == null) return null;
+		Object selItem = portList.getSelectedItem();
+		if (selItem == null) {
+			return null;
+		}
 		return portList.getSelectedItem().toString();
 	}