Nearing final release 2.2.13

help for data export
fixes to fx symbol opacity
removal of too much print output
change of error message when plugins loaded (don't say there is an error when a plugin is not compatible with that run mode)
Added option for SourcePanel to have a null return value (not used in any current detectors)

.classpath

@@ -6,8 +6,9 @@
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-17">
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER">
 		<attributes>
+			<attribute name="module" value="true"/>
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>

dependency-reduced-pom.xml

@@ -4,7 +4,7 @@
   <groupId>org.pamguard</groupId>
   <artifactId>Pamguard</artifactId>
   <name>Pamguard</name>
-  <version>2.02.12</version>
+  <version>2.02.13</version>
   <description>Pamguard using Maven to control dependencies</description>
   <url>www.pamguard.org</url>
   <organization>

pom.xml

@@ -5,7 +5,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.pamguard</groupId>
   <artifactId>Pamguard</artifactId>
-  <version>2.02.12</version>
+  <version>2.02.13</version>
   <name>Pamguard</name>
   <description>Pamguard using Maven to control dependencies</description>
 	<url>www.pamguard.org</url>
@@ -579,8 +579,7 @@
 			<version>0.1.55</version>
 		</dependency>
 
-		<!-- https://mvnrepository.com/artifact/com.fazecast/jSerialComm 
+		<!-- https://mvnrepository.com/artifact/com.fazecast/jSerialComm -->
-I've tried updating this to 2.11 bu tit's throwing problems at me with a missing Android class-->
 		<dependency>
 			<groupId>com.fazecast</groupId>
 			<artifactId>jSerialComm</artifactId>

src/Acquisition/FileInputSystem.java

@@ -595,7 +595,7 @@ public class FileInputSystem  extends DaqSystem implements ActionListener, PamSe
 			System.out.println("The current file was null");
 			return false;
 		}
-		System.out.printf("***********************************             Opening file %s\n", currentFile.getName());
+//		System.out.printf("***********************************             Opening file %s\n", currentFile.getName());
 
 		try {
 

src/GPS/GPSParametersDialog.java

@@ -183,10 +183,10 @@ public class GPSParametersDialog extends PamDialog {
 		}
 		PamDataBlock gpsSource = sourcePanel.getSource();
 		if (gpsSource == null && isNormal) {
-			return false;
+//			return false;
 		}
 		else if (gpsSource != null) {
-			gpsParameters.nmeaSource = gpsSource.getDataName();
+			gpsParameters.nmeaSource = gpsSource.getLongDataName();
 		}
 		else {
 			gpsParameters.nmeaSource = null;

src/GPS/ProcessNmeaData.java

@@ -127,7 +127,7 @@ public class ProcessNmeaData extends PamProcess {
 			return false;
 		}
 		
-		newDataBlock = PamController.getInstance().getDataBlock(NMEADataUnit.class, gpsController.gpsParameters.nmeaSource);
+		newDataBlock = PamController.getInstance().getDataBlockByLongName(gpsController.gpsParameters.nmeaSource);
 		if (newDataBlock == null) {
 			newDataBlock = PamController.getInstance().getDataBlock(NMEADataUnit.class, 0);
 		}

src/PamController/PamguardVersionInfo.java

@@ -31,12 +31,12 @@ public class PamguardVersionInfo {
 	 * Version number, major version.minorversion.sub-release.
 	 * Note: can't go higher than sub-release 'f'
 	 */
-	static public final String version = "2.02.12";
+	static public final String version = "2.02.13";
 	
 	/**
 	 * Release date
 	 */
-	static public final String date = "19 August 2024";
+	static public final String date = "26 August 2024";
 	
 //	/**
 //	 * Release type - Beta or Core

src/PamModel/PamModel.java

@@ -1238,18 +1238,16 @@ final public class PamModel implements PamSettings {
 									}
 
 									// only add the plugin to the list if this is a valid run mode
-									if (pf.allowedModes()==PamPluginInterface.ALLMODES ||
+//									if (isAllowedMode(pf)) {
-											(pf.allowedModes()==PamPluginInterface.VIEWERONLY && isViewer ) ||
-											(pf.allowedModes()==PamPluginInterface.NOTINVIEWER && !isViewer)) {
 										pf.setJarFile(jarName);	// save the name of the jar, so that javahelp can find the helpset
 										if (getPluginBeingLoaded()==null) {
 											continue;
 										}
 
 										pluginList.add(pf); // add it to the list
-									} else {
+//									} else {
-										System.out.println("     Error: " + pf.getDefaultName()+" cannot run in this mode.  Skipping module.");									
+//										System.out.println("     Warning: " + pf.getDefaultName()+" cannot run in this mode.  Skipping module.");									
-									}
+//									}
 									if (getPluginBeingLoaded()==null) {
 										continue;
 									}
@@ -1337,6 +1335,10 @@ final public class PamModel implements PamSettings {
 							//URLClassLoader cl = new URLClassLoader(new URL[]{classFile.toURI().toURL()});
 //							mi = PamModuleInfo.registerControlledUnit(pf.getClassName(), pf.getDescription(),cl);
 							mi = PamModuleInfo.registerControlledUnit(pf.getClassName(), pf.getDescription(),classLoader);
+							if (isAllowedMode(pf) == false) {
+								mi.setHidden(true);
+								System.out.println("     Warning: " + pf.getDefaultName()+" cannot run in this mode.  hiding module.");	
+							}
 						} catch (Exception e) {
 							System.err.println("   Error accessing " + pf.getJarFile());
 							e.printStackTrace();
@@ -1420,6 +1422,17 @@ final public class PamModel implements PamSettings {
 		this.clearPluginBeingLoaded();
 	}
 
+	/**
+	 * Can this plugin run in this mode ? 
+	 * Even if it can't, leave it in the model, but don't allow any to be created. 
+	 * @param plugin
+	 * @return
+	 */
+	private boolean isAllowedMode(PamPluginInterface pf) {
+		return (pf.allowedModes()==PamPluginInterface.ALLMODES ||
+				(pf.allowedModes()==PamPluginInterface.VIEWERONLY && isViewer ) ||
+				(pf.allowedModes()==PamPluginInterface.NOTINVIEWER && !isViewer));
+	}
 	/**
 	 * Return a list of the plugins found in the plugin folder
 	 * @return

src/PamView/dialog/SourcePanel.java

@@ -28,6 +28,8 @@ import PamDetection.LocalisationInfo;
 import PamUtils.PamUtils;
 import PamguardMVC.PamConstants;
 import PamguardMVC.PamDataBlock;
+import PamguardMVC.PamDataUnit;
+import PamguardMVC.PamProcess;
 
 /**
  * Standard panel for dialogs that shows a list of
@@ -46,7 +48,7 @@ public class SourcePanel implements ActionListener{
 	protected String borderTitle;
 	protected boolean includeSubClasses;
 	
-	protected JComboBox sourceList;
+	protected JComboBox<PamDataBlock> sourceList;
 	protected JCheckBox channelBoxes[];
 	
 	protected Window ownerWindow;
@@ -66,6 +68,10 @@ public class SourcePanel implements ActionListener{
 	
 	private JLabel channelListHeader;
 	
+	/**
+	 * Allow a null selection, i.e. don't want to select anything at all. 
+	 */
+	private boolean allowNull;
 	/**
 	 * Construct a panel with a titles border
 	 * @param ownerWindow parentWindow
@@ -253,6 +259,7 @@ public class SourcePanel implements ActionListener{
 	}
 	
 	private int currentNShown = 0;
+	private NullDataBlock nullDataBlock;
 	/**
 	 * Repack the owner window if the number of channels has changed
 	 * @param channelsMap bitmap of used channels. 
@@ -386,6 +393,10 @@ public class SourcePanel implements ActionListener{
 		
 		LocalisationInfo availableLocData;
 		
+		if (allowNull) {
+			sourceList.addItem(getNullDataBlock());
+		}
+		
 		for (int i = 0; i < sl.size(); i++) {
 			
 			if (excludedBlocks.contains(sl.get(i))) continue;
@@ -432,7 +443,13 @@ public class SourcePanel implements ActionListener{
 	 * @return source data block
 	 */
 	public PamDataBlock getSource() {
-		return (PamDataBlock) sourceList.getSelectedItem();
+		PamDataBlock source = (PamDataBlock) sourceList.getSelectedItem();
+		if (source == getNullDataBlock()) {
+			return null;
+		}
+		else {
+			return source;
+		}
 	}
 	
 	/**
@@ -583,4 +600,40 @@ public class SourcePanel implements ActionListener{
 	public void setSourceToolTip(String toolTip) {
 		sourceList.setToolTipText(toolTip);
 	}
+
+	/**
+	 * @return the allowNull
+	 */
+	public boolean isAllowNull() {
+		return allowNull;
+	}
+
+	/**
+	 * Allow a null selection. 
+	 * @param allowNull the allowNull to set
+	 */
+	public void setAllowNull(boolean allowNull) {
+		this.allowNull = allowNull;
+		setSourceList();
+	}
+	
+	private NullDataBlock getNullDataBlock() {
+		if (nullDataBlock == null) {
+			nullDataBlock = new NullDataBlock();
+		}
+		return nullDataBlock;
+	}
+	
+	private class NullDataBlock extends PamDataBlock {
+
+		public NullDataBlock() {
+			super(PamDataUnit.class, "Null data", null, 0);
+		}
+
+		@Override
+		public String toString() {
+			return "No (null) selection";
+		}
+		
+	}
 }

src/dataMap/OfflineDataMap.java

@@ -565,10 +565,10 @@ abstract public class OfflineDataMap<TmapPoint extends OfflineDataMapPoint> {
 
 	private void warnDoubleFind(long timeMillis, TmapPoint foundPoint,
 			TmapPoint mapPoint) {
-		System.out.println(String.format("Warning - data unit at time %s found in two map points %s - %s and %s - %s in %s ",
+//		System.out.println(String.format("Warning - data unit at time %s found in two map points %s - %s and %s - %s in %s ",
-				PamCalendar.formatDateTime(timeMillis), PamCalendar.formatDateTime(foundPoint.getStartTime()),
+//				PamCalendar.formatDateTime(timeMillis), PamCalendar.formatDateTime(foundPoint.getStartTime()),
-				PamCalendar.formatDateTime(foundPoint.getEndTime()),PamCalendar.formatDateTime(mapPoint.getStartTime()),
+//				PamCalendar.formatDateTime(foundPoint.getEndTime()),PamCalendar.formatDateTime(mapPoint.getStartTime()),
-				PamCalendar.formatDateTime(mapPoint.getEndTime()), this.parentDataBlock.getDataName()));
+//				PamCalendar.formatDateTime(mapPoint.getEndTime()), this.parentDataBlock.getDataName()));
 	}
 
 	/**

src/generalDatabase/EmptyTableDefinition.java

@@ -402,7 +402,13 @@ public class EmptyTableDefinition implements Cloneable {
 	protected EmptyTableDefinition clone() {
 		try {
 			EmptyTableDefinition clone = (EmptyTableDefinition) super.clone();
-//			clone.pamTableItems = new ArrayList<>();
+			/**
+			 * Don't clone the list items themselves since these may be referenced from the
+			 * SQLLogging class, which won't be able to find them if their references change.
+			 * Just clone the array list, leaving the individual items alone.  
+			 */
+			clone.pamTableItems = new ArrayList<>();
+			clone.pamTableItems.addAll(this.pamTableItems);
 //			for (PamTableItem tableItem : this.pamTableItems) {
 //				clone.pamTableItems.add(tableItem.clone());
 //			}

src/geoMag/MagneticVariation.java

@@ -1,6 +1,5 @@
 package geoMag;
 
-import weka.core.SingleIndex;
 import GPS.GpsData;
 
 /**

src/help/JavaHelpSearch/SCHEMA

@@ -1,2 +1,2 @@
 JavaSearch 1.0
-TMAP bs=2048 rt=1 fl=-1 id1=6905 id2=1
+TMAP bs=2048 rt=1 fl=-1 id1=6961 id2=1

src/help/Map.jhm

@@ -198,6 +198,8 @@
 
 <mapID target="utilities.quickAnnotations.docs.quickAnnotations" url="utilities/quickAnnotations/docs/quickAnnotations.html"/>
 
+<mapID target="overview.dataexport.docs.dataexport" url="overview/dataexport/docs/dataexport.html"/>
+
 <mapID target="classifiers.roccaHelp.docs.rocca_Spectrogram" url="classifiers/roccaHelp/docs/rocca_Spectrogram.html"/>
 
 <mapID target="displays.userDisplayHelp.docs.userDisplayPanel" url="displays/userDisplayHelp/docs/userDisplayPanel.html"/>

src/help/PAMGUARDTOC.xml

@@ -63,6 +63,10 @@
    <tocitem text="Target Motion Annotation " target="overview.annotations.docs.targetmotionannotation" image="topic"/>
 
   </tocitem>
+  <tocitem text="Data Export ">
+   <tocitem text="Data Export " target="overview.dataexport.docs.dataexport" image="topic"/>
+
+  </tocitem>
  </tocitem>
  <tocitem text="Maps and Mapping ">
   <tocitem text="NMEA and GPS ">

src/help/overview/dataexport/docs/dataexport.html

@@ -0,0 +1,91 @@
+<html>
+<head>
+<LINK href="../../../pamHelpStylesheet.css" type="text/css"
+	rel="STYLESHEET">
+<title>Data Annotation</title>
+</head>
+
+<body>
+<h1>PAMGuard exporter</h1>
+<h2>Introduction</h2>
+<p>The PAMGuard exporter allows users to export PAMGuard data, such as detections, to a variety of different formats. The exporter is a convenient solution for exporting sections or large chunks of a PAMGuard datasets without requiring any code. For more bespoke data management please see the <a href="https://github.com/PAMGuard/PAMGuardMatlab">PAMGuard-MATLAB</a> library and <a href="https://github.com/TaikiSan21/PamBinaries">PAMBinaries package</a> which can be used for more bespoke data management. Note that the exporter only exports a sub set of data types - this will expand in future releases.</p>
+<h2>Exporting</h2>
+<p>The PAMGuard exporter can be accessed from <em>File-&gt;Export</em>. This brings up the Export dialog. The export dialog allows users to select which data to export, where to export it and the file format to export as. Each data block also has a settings icon which opens the data block's unique data selector. So for example, users can export only specific types of clicks or whistles between certain frequencies.</p>
+<p align="center">
+  <img width="920" height="475" src = "resources/PAMGuard_exporter_dialog_annotated.png">
+</p>
+<center><em>  Diagram of the exporter dialog. The dialog allows users to select which part of the dataset to export, how to export it and which type of data to export </em></center>
+<p>The main parts of the dialog are as follows.</p>
+<h3>Data Options</h3>
+<p>Select which part of the dataset to export</p>
+<ul>
+<li>Loaded Data : the data currently loaded into memory i.e. usually what you can see in the displays - may be different time oeriods depending on the data type.</li>
+<li>All data : the entire dataset.</li>
+<li>Select data :  manually enter a period between two times.</li>
+<li>Specify time chunks : import a csv file with a list of time chunks.</li>
+</ul>
+<h3>Export Options</h3>
+<p>Select where to export the data to using <em>Browse...</em> and select the maximum allowed file size using the <em>Maximum file size</em> selector. Select the format by toggling one of the data format buttons. Hover over each button to see more info.</p>
+<h3>Export Data</h3>
+<p>Select which data to export. If a data type has a cog icon next to it then it has a data selector. The data selector settings can be used to filter which detections are exported. For example you may wish only to export clicks of a certain type or perhaps deep learning detections with a prediction value above a certain threshold. Each data selector is unique to the type of data. Note that the exporter only exports a sub set of data types - this will expand in future.</p>
+<h3>Progress</h3>
+<p>Once <em>Start</em> is selected then the progress bars show progress in exporting the selected data.</p>
+<h2>Export formats</h2>
+<p>Currently the exporter has three possible output formats.</p>
+<h3>MAT files</h3>
+<p>MAT files are files which can be opened easily in MATLAB and Python. They can store multiple different data formats e.g. tables, arrays, structures. Each PAMGuard detection is saved as a single structure and then the file contains an array of these structures for each data type. The fields within the structure contains the relevant data unique to each data unit. Whilst data units have unique fields depending on their type e.g. a click or a whistle, there are some fields that are shared between almost all data units - an example of a click detection structure is shown below</p>
+<p><em>General fields shared by most data units in PAMGuard</em></p>
+<ul>
+<li><em>millis</em>: the unix*1000 start time of the click, whistle, clip etc. in milliseconds; this number can be converted to a date/time with millisecond accuracy.</li>
+<li><em>date</em>: the start time of the click in MATLAB datenum format. Use datastr(date) to show a time string.</li>
+<li><em>UID</em>:  a unique serial number for the detection. Within a processed dataset no other detection will have this number.</li>
+<li><em>startSample</em>: The first sample of this detection - often used for finer scale time delay measurements. Samples refers to the number of samples in total the sound card has taken since processing begun or a new file has been created.</li>
+<li><em>channelMap</em>: The channel map for this detection. One number which represents which channels this detection is from: To get the true channels use the getChannels(channelMap) function.</li>
+</ul>
+<p><em>Unique to clicks</em></p>
+<ul>
+<li><em>triggerMap</em>: which channel triggered the detection.</li>
+<li><em>type</em>: Classification type. Must use database or settings to see what species this refers to.</li>
+<li><em>duration</em>: Duration of this click detection in samples.</li>
+<li><em>nChan</em>: Number of channels the detection was made on.</li>
+<li><em>wave</em>: Waveform data for each channel.</li>
+</ul>
+<p>Note that the format of each struct is the same as the format if extracting data using the <a href="https://github.com/PAMGuard/PAMGuardMatlab">PAMGuard-MATLAB</a> library.</p>
+<p>To open an exported .mat file simply drag it into <strong>MATLAB</strong> or use the function;</p>
+<pre><code class="language-Matlab"> load(/my/path/to/file.mat)
+</code></pre>
+<p>To open a .mat file in <strong>Python</strong> use</p>
+<pre><code class="language-Python">import scipy.io
+mat = scipy.io.loadmat('/my/path/to/file.mat')
+clkstruct = mat['det_20170704_204536_580'] #The name of the struct array within the file
+ 
+#Extract the third waveform from a click example
+nwaves = len(clkstruct[0]) #Number of clicks
+thirdwaveform = clkstruct[0, 2]['wave'] #Waveform from third click in samples between -1 and 1. 
+</code></pre>
+<h3>R</h3>
+<p>Data can be exported to an RData frame. The data are exported as R structs with the same fields as in MATLAB (and PAMBinaries package). To open a an RData frame open RStudio and import the file or use;</p>
+<pre><code class="language-R">load(&quot;/my/path/to/file.RData&quot;)
+</code></pre>
+<h3>Wav files</h3>
+<p>Any detection which contains raw sound data, for example a click, clip or deep learning detection, can be exported as a wav file. When wav files are selected three options are presented for saving files.</p>
+<p align="center">
+  <img width="300" height="450" src = "resources/PAMGuard_exporter_dialog_wav.png">
+</p>
+<center><em> When wav files are selected additional options are presented on how to save the file </em></center>
+<ul>
+<li>
+<p><em>Zero pad</em> :  Here detections are saved as wav files with the time in between detections zero padded. The resulting files will be as large as the initial wav files processed to create the data. This can be useful if for example opening the files in another acoustic analysis program.</p>
+</li>
+<li>
+<p><em>Concatenate</em> : The detections are saved to a wav file without any zero padding. This saves storage space but temporal information is lost within the wav file. The sample positions of each detection are saved in a text file along with the wav file so that temporal info is available if needed. This is same format as SoundTrap click detection data.</p>
+</li>
+<li>
+<p><em>Individual</em> : Each detection is saved in it's own time stamped individual sound file.</p>
+</li>
+</ul>
+<h2>After export</h2>
+<p>Once data are exported, the exported files are not part of PAMGuard's data management system i.e. PAMGuard has no record they exist and they are not shown in the data model etc. If you export the same data again to the same location, then previous exported files may be overwritten without warning.</p>
+
+</body>
+</html>

src/rawDeepLearningClassifier/segmenter/SegmenterProcess.java

@@ -736,7 +736,7 @@ public class SegmenterProcess extends PamProcess {
 
 		//add some extra metadata to the chunks 
 		packageSegmenterDataUnit(currentRawChunks[i]); 
-		//System.out.println("Segmenter process: Save current segments to datablock: " + currentRawChunks[i].getParentDataUnit().getUID() + " " + i + currentRawChunks[i].getRawData()[0][0]); 
+//		System.out.println("Segmenter process: Save current segments to datablock: " + currentRawChunks[i].getParentDataUnit().getUID() + " " + i + currentRawChunks[i].getRawData()[0][0]); 
 
 		//send the raw data unit off to be classified!