Exporter now working

.classpath

@@ -6,7 +6,7 @@
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/Amazon Coretto 21">
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.launching.macosx.MacOSXType/Amazon Corretto 21 [21.0.2]">
 		<attributes>
 			<attribute name="module" value="true"/>
 			<attribute name="maven.pomderived" value="true"/>

.settings/org.eclipse.core.resources.prefs

@@ -1,5 +1,6 @@
 eclipse.preferences.version=1
 encoding//src/rawDeepLearningClassifer/segmenter/SegmenterProcess.java=UTF-8
+encoding//src/test=UTF-8
 encoding//src/test/resources=UTF-8
 encoding/<project>=UTF-8
 encoding/src=UTF-8

.settings/org.eclipse.jdt.core.prefs

@@ -1,9 +1,9 @@
 eclipse.preferences.version=1
 org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
 org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=18
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=21
 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=18
+org.eclipse.jdt.core.compiler.compliance=21
 org.eclipse.jdt.core.compiler.debug.lineNumber=generate
 org.eclipse.jdt.core.compiler.debug.localVariable=generate
 org.eclipse.jdt.core.compiler.debug.sourceFile=generate
@@ -13,4 +13,4 @@ org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
 org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
 org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning
 org.eclipse.jdt.core.compiler.release=enabled
-org.eclipse.jdt.core.compiler.source=18
+org.eclipse.jdt.core.compiler.source=21

src/PamModel/PamModel.java

@@ -24,7 +24,6 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.Serializable;
 import java.lang.reflect.Constructor;
-import java.lang.reflect.Method;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.util.ArrayList;
@@ -51,8 +50,6 @@ import rockBlock.RockBlockControl;
 import tethys.TethysControl;
 import turbineops.TurbineOperationControl;
 import GPS.GpsDataUnit;
-import Map.MapController;
-import Map.gridbaselayer.GridbaseControl;
 import NMEA.NMEADataUnit;
 import PamController.PamControlledUnitSettings;
 import PamController.PamController;
@@ -66,7 +63,6 @@ import PamguardMVC.PamDataBlock;
 import analogarraysensor.ArraySensorControl;
 import backupmanager.BackupManager;
 import beamformer.continuous.BeamFormerControl;
-import beamformer.localiser.BeamFormLocaliserControl;
 import bearinglocaliser.BearingLocaliserControl;
 import binaryFileStorage.SecondaryBinaryStore;
 import cepstrum.CepstrumControl;

src/export/CSVExport/CSVExportManager.java

@@ -63,5 +63,11 @@ public class CSVExportManager implements PamDataUnitExporter{
 		// TODO Auto-generated method stub
 		return null;
 	}
+	
+	
+	@Override
+	public void prepareExport() {
+		
+	}
 
 }

src/export/MLExport/MLDetectionsManager.java

@@ -286,7 +286,11 @@ public class MLDetectionsManager implements PamDataUnitExporter {
 		return null;
 	}
 
-
+	
+	@Override
+	public void prepareExport() {
+		this.currentFile = null; 
+	}
 
 
 }

src/export/MLExport/MLWhistleMoanExport.java

@@ -47,10 +47,10 @@ public class MLWhistleMoanExport extends MLDataUnitExport<ConnectedRegionDataUni
 
 	/**
 	 * Calculate summary contour information to allow people to conveniently plot whisltle data
-	 * @param dataUnit - the connected regio data unit. 
+	 * @param dataUnit - the connected region data unit. 
 	 * @return peak contour frequency int[0] and contour width int[1]. 
 	 */
-	private int[][] calcPeakContourWidths(ConnectedRegionDataUnit dataUnit){
+	public static int[][] calcPeakContourWidths(ConnectedRegionDataUnit dataUnit){
 		
 		SliceData sliceData; 
 		// should change this to use iterators, not get since sliceList is linked list. 

src/export/PamDataUnitExporter.java

@@ -69,7 +69,12 @@ public interface PamDataUnitExporter {
 	 * An optional JavaFX pane that displays additional options for the user.
 	 * @return pane with additonal options - can be null. 
 	 */
-	public Pane getOptionsPane(); 
+	public Pane getOptionsPane();
+
+	/**
+	 * Called whenever a new export run is prepared. 
+	 */
+	public void prepareExport(); 
 
 
 }

src/export/PamExporterManager.java

@@ -188,6 +188,13 @@ public class PamExporterManager {
 		return this.pamExporters.get(this.exportParams.exportChoice);
 	}
 
+	public void perpareExport() {
+		for (PamDataUnitExporter exporter:pamExporters) {
+			exporter.prepareExport();
+		}
+		
+	}
+
 
 
 

src/export/RExport/RExportManager.java

@@ -55,9 +55,11 @@ public class RExportManager implements PamDataUnitExporter {
 
 	@Override
 	public boolean exportData(File fileName, List<PamDataUnit> dataUnits, boolean append) {
+		
+		if (dataUnits==null || dataUnits.size()<=0) return false;
 
 		/**
-		 * Note - there is no way to save data units to R files wothout loading the file into memory. 
+		 * Note - there is no way to save data units to R files without loading the file into memory. 
 		 * So everything is stored in memory until saved. 
 		 */
 		PamDataUnit minByTime = PamArrayUtils.getMinTimeMillis(dataUnits);
@@ -269,7 +271,7 @@ public class RExportManager implements PamDataUnitExporter {
 		
 		//TODO 
 		//check file size against the export params. 
-		System.out.println("RData length: " + allData.length());
+//		System.out.println("RData length: " + allData.length());
 		
 		return false;
 	}
@@ -287,6 +289,11 @@ public class RExportManager implements PamDataUnitExporter {
 		// TODO Auto-generated method stub
 		return null;
 	}
+	
+	@Override
+	public void prepareExport() {
+		
+	}
 
 
 

src/export/RExport/RWhistleExport.java

@@ -1,8 +1,17 @@
 package export.RExport;
 
+import org.renjin.sexp.AttributeMap;
+import org.renjin.sexp.IntArrayVector;
+import org.renjin.sexp.ListVector;
+import export.MLExport.MLWhistleMoanExport;
+import us.hebi.matlab.mat.format.Mat5;
+import us.hebi.matlab.mat.types.Struct;
+
 import org.renjin.sexp.ListVector.NamedBuilder;
 
+import PamUtils.PamArrayUtils;
 import whistlesAndMoans.ConnectedRegionDataUnit;
+import whistlesAndMoans.SliceData;
 
 /***
  * Export whisltes to RData
@@ -13,8 +22,27 @@ public class RWhistleExport extends RDataUnitExport<ConnectedRegionDataUnit> {
 
 	@Override
 	public NamedBuilder addDetectionSpecificFields(NamedBuilder rData, ConnectedRegionDataUnit dataUnit, int index) {
+	
+		
+		rData.add("nSlices", dataUnit.getConnectedRegion().getNumSlices()); 
+		
+		int[][] contourData = MLWhistleMoanExport.calcPeakContourWidths( dataUnit); 
+
+		IntArrayVector contours = new IntArrayVector(contourData[0]);
+		IntArrayVector contourWidth = new IntArrayVector(contourData[0]);
+		
+		//need to generate a slice struct
+		ListVector.NamedBuilder peakDatas = createSliceStruct( dataUnit);
+
+		rData.add("nSlices", dataUnit.getConnectedRegion().getNumSlices()); 
+		rData.add("sliceData", peakDatas); 
+		rData.add("contour", contours); 
+		rData.add("contourWidth", contourWidth); 
+		rData.add("meanWidth", PamArrayUtils.mean(contourData[0])); 
+
+	
 		// TODO Auto-generated method stub
-//		//nSlices int
+		//nSlices int
 //		MLInt32 nSlices = new MLInt32(null, new Integer[]{dataUnit.getConnectedRegion().getNumSlices()}, 1);
 //		
 //		//list of structures: sliceNumber int, nPeaks int, peakData
@@ -42,6 +70,71 @@ public class RWhistleExport extends RDataUnitExport<ConnectedRegionDataUnit> {
 		return rData;
 	}
 	
+	
+	
+	/**
+	 * Create array of slice structures for output.
+	 * @param dataUnit
+	 * @return
+	 */
+	private ListVector.NamedBuilder createSliceStruct(ConnectedRegionDataUnit dataUnit){
+		
+//		Struct mlStructure= new MLStructure("sliceData", new int[]{dataUnit.getConnectedRegion().getSliceData().size(), 1}); 
+		
+		ListVector.NamedBuilder peakDatas = new ListVector.NamedBuilder(); ;
+
+		
+		Struct mlStructure= Mat5.newStruct(dataUnit.getConnectedRegion().getSliceData().size(), 1);
+
+		//the start sample.
+		int sliceNumber;
+		int nPeaks;
+		int[][] peakData;
+		SliceData sliceData; 
+		ListVector.NamedBuilder rData;
+		for (int i=0; i<dataUnit.getConnectedRegion().getSliceData().size(); i++){
+			
+			rData = new ListVector.NamedBuilder();
+			
+			//slice data
+			sliceData= dataUnit.getConnectedRegion().getSliceData().get(i); 
+		
+			//the start sample.
+			sliceNumber =  sliceData.getSliceNumber(); 
+
+			//the duration of the detection in samples.
+			nPeaks = sliceData.getnPeaks(); 
+
+			//the frequency limits.
+			peakData = sliceData.getPeakInfo(); 
+			
+			int n=0;
+			int nbins =peakData.length*peakData[0].length;
+			int[] flattenedArr  = new int[nbins];
+			//System.out.println("Number of bins: " + nbins);
+			for (int ii=0; ii<peakData.length; ii++) {
+				for (int j=0; j<peakData[ii].length; j++) {
+//					System.out.println("Current: " + i + " "+ j 
+//							+ " nchan: " + dataUnit.getNChan() + "  wave size: " 
+//							+ dataUnit.getWaveLength() +"len concat: " + concatWaveform.length);
+					flattenedArr[n++] = peakData[ii][j];
+				}
+			}
+
+			IntArrayVector peakDataR = new IntArrayVector(flattenedArr, AttributeMap.builder().setDim(peakData.length, peakData[0].length).build());
+					
+			rData.add("sliceNumber", sliceNumber);
+			rData.add("nPeaks", nPeaks);
+			rData.add("peakData", peakDataR);
+			
+			peakDatas.add(String.valueOf(sliceNumber), rData); 
+
+		}
+		
+		return peakDatas;
+		
+	}
+	
 	@Override
 	public Class<?> getUnitClass() {
 		return ConnectedRegionDataUnit.class;

src/export/exporter_help.md

@@ -0,0 +1,94 @@
+# PAMGuard exporter
+## Introduction
+
+The PAMGuard exporter allows users to export PAMGuard data, such as detections, to a variety of different formats. The exporter is a convenient solution for exporting sections or large chunks of a PAMGuard datasets without requiring any code. For more bespoke data management please see the [PAMGuard-MATLAB](https://github.com/PAMGuard/PAMGuardMatlab) library and [PAMBinaries package](https://github.com/TaikiSan21/PamBinaries) which can be used for more bespoke data management. Note that the exporter only exports a sub set of data types - this will expand in future releases. 
+
+## Exporting
+The PAMGuard exporter can be accessed from *File->Export*. This brings up the Export dialog. The export dialog allows users to select which data to export, where to export it and the file format to export as. Each data block also has a settings icon which opens the data block's unique data selector. So for example, users can export only specific types of clicks or whistles between certain frequencies. 
+
+<p align="center">
+  <img width="920" height="450" src = "resources/PAMGuard_exporter_dialog_annotated.png">
+</p>
+
+<center><em>  Diagram of the exporter dialog. The dialog allows users to select which part of the dataset to export, how to export it and which type of data to export </em></center>
+
+The main parts of the dialog are as follows. 
+### Data Options
+Select which part of the dataset to export
+- Loaded Data : the data currently loaded into memory i.e. usually what you can see in the displays - may be different time oeriods depending on the data type. 
+- All data : the entire dataset. 
+- Select data :  manually enter a period between two times. 
+- Specify time chunks : import a csv file with a list of time chunks.
+
+### Export Options
+Select where to export the data to using _Browse..._ and select the maximum allowed file size using the _Maximum file size_ selector. Select the format by toggling one of the data format buttons. Hover over each button to see more info. 
+
+### Export Data
+Select which data to export. If a data type has a cog icon next to it then it has a data selector. The data selector settings can be used to filter which detections are exported. For example you may wish only to export clicks of a certain type or perhaps deep learning detections with a prediction value above a certain threshold. Each data selector is unique to the type of data. Note that the exporter only exports a sub set of data types - this will expand in future. 
+
+### Progress
+Once _Start_ is selected then the progress bars show progress in exporting the selected data. 
+
+## Export formats
+Currently the exporter has three possible output formats.
+
+### MAT files
+MAT files are files which can be opened easily in MATLAB and Python. They can store multiple different data formats e.g. tables, arrays, structures. Each PAMGuard detection is saved as a single structure and then the file contains an array of these structures for each data type. The fields within the structure contains the relevant data unique to each data unit. Whilst data units have unique fields depending on their type e.g. a click or a whistle, there are some fields that are shared between almost all data units - an example of a click detection structure is shown below
+
+*General fields shared by most data units in PAMGuard*
+- *millis*: the unix*1000 start time of the click, whistle, clip etc. in milliseconds; this number can be converted to a date/time with millisecond accuracy.
+- *date*: the start time of the click in MATLAB datenum format. Use datastr(date) to show a time string. 
+- *UID*:  a unique serial number for the detection. Within a processed dataset no other detection will have this number. 
+- *startSample*: The first sample of this detection - often used for finer scale time delay measurements. Samples refers to the number of samples in total the sound card has taken since processing begun or a new file has been created. 
+- *channelMap*: The channel map for this detection. One number which represents which channels this detection is from: To get the true channels use the getChannels(channelMap) function.
+
+*Unique to clicks*
+- *triggerMap*: which channel triggered the detection.
+- *type*: Classification type. Must use database or settings to see what species this refers to. 
+- *duration*: Duration of this click detection in samples. 
+- *nChan*: Number of channels the detection was made on. 
+- *wave*: Waveform data for each channel. 
+
+Note that the format of each struct is the same as the format if extracting data using the [PAMGuard-MATLAB](https://github.com/PAMGuard/PAMGuardMatlab) library. 
+
+To open an exported .mat file simply drag it into **MATLAB** or use the function;
+```Matlab
+ load(/my/path/to/file.mat)
+``` 
+
+To open a .mat file in **Python** use 
+
+```Python
+import scipy.io
+mat = scipy.io.loadmat('/my/path/to/file.mat')
+clkstruct = mat['det_20170704_204536_580'] #The name of the struct array within the file
+ 
+#Extract the third waveform from a click example
+nwaves = len(clkstruct[0]) #Number of clicks
+thirdwaveform = clkstruct[0, 2]['wave'] #Waveform from third click in samples between -1 and 1. 
+```
+
+### R
+Data can be exported to an RData frame. The data are exported as R structs with the same fields as in MATLAB (and PAMBinaries package). To open a an RData frame open RStudio and import the file or use;
+
+```R
+load("/my/path/to/file.RData")
+```
+
+### Wav files
+Any detection which contains raw sound data, for example a click, clip or deep learning detection, can be exported as a wav file. When wav files are selected three options are presented for saving files. 
+
+<p align="center">
+  <img width="300" height="450" src = "resources/PAMGuard_exporter_dialog_wav.png">
+</p>
+
+<center><em> When wav files are selected additional options are presented on how to save the file </em></center>
+
+- *Zero pad* :  Here detections are saved as wav files with the time in between detections zero padded. The resulting files will be as large as the initial wav files processed to create the data. This can be useful if for example opening the files in another acoustic analysis program. 
+
+- *Concatenate* : The detections are saved to a wav file without any zero padding. This saves storage space but temporal information is lost within the wav file. The sample positions of each detection are saved in a text file along with the wav file so that temporal info is available if needed. This is same format as SoundTrap click detection data. 
+
+- *Individual* : Each detection is saved in it's own time stamped individual sound file.
+
+## After export
+Once data are exported, the exported files are not part of PAMGuard's data management system i.e. PAMGuard has no record they exist and they are not shown in the data model etc. If you export the same data again to the same location, then previous exported files may be overwritten without warning. 

src/export/swing/ExportProcessDialog.java

@@ -255,6 +255,8 @@ public class ExportProcessDialog {
 			mainPanel.add(extraSettingsPanel);
 			//add the main panel at a different index. 
 			getMainPanel().add(mainPanel, 1);
+			
+			this.getTasksPanel().setBorder(new TitledBorder("Export Data"));
 
 			pack();
 
@@ -410,12 +412,14 @@ public class ExportProcessDialog {
 			@Override
 			public void setTaskStatus(TaskMonitorData taskMonitorData) {
 				
+				//System.out.println();			
 				switch (taskMonitorData.taskStatus) {
 				case COMPLETE:
 					if (taskIndex<exportTaskGroup.getNTasks() && !started) {
 						exportTaskGroup.runTaskFrom(taskIndex+1);
 						started = true;
 					}
+					else super.setTaskStatus(taskMonitorData);
 					break;
 				default:
 					super.setTaskStatus(taskMonitorData);
@@ -425,7 +429,6 @@ public class ExportProcessDialog {
 				switch (taskMonitorData.taskActivity) {
 				case LINKING:
 				case LOADING:
-					
 					//the progress of one datablocks
 					double progress = ((double) taskMonitorData.progValue)/taskMonitorData.progMaximum;
 					double totalProgress = ((double) taskIndex)/activeTasks + progress/activeTasks;

src/export/swing/ExportTask.java

@@ -37,6 +37,14 @@ public class ExportTask extends OfflineTask<PamDataUnit<?,?>>{
 
 
 	}
+	
+	/**
+	 * Called at the start of the thread which executes this task. 
+	 */
+	@Override
+	public void prepareTask() {
+		exporter.perpareExport();
+	}
 
 	@Override
 	public String getName() {

src/export/wavExport/WavDetExport.java

@@ -352,16 +352,15 @@ public class WavDetExport {
 						//now safety check - is this more than one GB of data. Each sample is 16bits but the input double array is 64 bits each. 
 						double size = samplesPad*16*audioFormat.getChannels()/1024/1024;
 						
+						System.out.println("Append wav. zero pad" + samplesPad);
 						//
 						if (size>MAX_ZEROPAD_SIZE_MEGABYTES) {
 							wavWrite.close();
-							System.err.println(String.format("WavExportManager: A zero padding of %.2f MB was requested. The maximum allowed size is %.2f - "
+							System.err.println(String.format("WavDetExport: A zero padding of %.2f MB was requested. The maximum allowed size is %.2f - "
 									+ "the .wav file was closed and any additional data units have not been written %s", size, MAX_ZEROPAD_SIZE_MEGABYTES, currentFile));
 							return n;
 						}
 						
-						System.out.println("Append wav. zero pad" + samplesPad);
-
 						wavWrite.append(new double[audioFormat.getChannels()][samplesPad]);
 					}
 					

src/export/wavExport/WavDetExportManager.java

@@ -60,6 +60,8 @@ public class WavDetExportManager implements PamDataUnitExporter  {
 	 */
 	private WavDetExport wavDetExport = new WavDetExport();
 
+	private File currentFile;
+
 	public WavDetExportManager() {
 
 	}
@@ -76,6 +78,21 @@ public class WavDetExportManager implements PamDataUnitExporter  {
 	@Override
 	public boolean exportData(File fileName,
 			List<PamDataUnit> dataUnits, boolean append) {
+		
+		if (fileName==null) return false;
+		
+		
+		if (this.currentFile==null || this.currentFile.compareTo(fileName)!=0) {
+			//we have a new .wav file to create. 
+			if (fileName.exists()) {
+				//we need to delete it 
+				System.out.println("PAMGuard export: wav file already existed and has been deleted: " + fileName.getName()); 
+				fileName.delete();
+			}
+		}
+		
+		this.currentFile = fileName;
+		
 
 		//make sure we have the latest options. 
 		if (wavOptionsPanel!=null) {
@@ -156,6 +173,12 @@ public class WavDetExportManager implements PamDataUnitExporter  {
 		return null;
 	}
 
+	@Override
+	public void prepareExport() {
+		this.currentFile = null;
+		
+	}
+
 
 
 

src/export/wavExport/WavOptionsPanel.java

@@ -42,7 +42,7 @@ public class WavOptionsPanel extends PamPanel {
 		
 		c.gridx++;
 		
-		addComponent(this, noZeroPad = new JRadioButton("Concatonate"), c);
+		addComponent(this, noZeroPad = new JRadioButton("Concatenate"), c);
 
 		c.gridx++;
 		addComponent(this, indvidualWav = new JRadioButton("Individual"), c);

src/offlineProcessing/OLProcessDialog.java

@@ -2,6 +2,7 @@ package offlineProcessing;
 
 import java.awt.BorderLayout;
 import java.awt.Color;
+import java.awt.Container;
 import java.awt.GridBagConstraints;
 import java.awt.GridBagLayout;
 import java.awt.Insets;
@@ -112,6 +113,10 @@ public class OLProcessDialog extends PamDialog {
 	 */
 	private boolean isNeedaNote = true;
 
+	/**
+	 * Tasks panel
+	 */
+	private PamAlignmentPanel tasksPanel;
 
 	public OLProcessDialog(Window parentFrame, OfflineTaskGroup taskGroup, String title) {
 		super(parentFrame, title, false);
@@ -150,7 +155,7 @@ public class OLProcessDialog extends PamDialog {
 		dataSelectPanel.add(BorderLayout.SOUTH, southPanel);
 				
 
-		JPanel tasksPanel = new PamAlignmentPanel(BorderLayout.WEST);
+		tasksPanel = new PamAlignmentPanel(BorderLayout.WEST);
 		tasksPanel.setLayout(new GridBagLayout());
 		tasksPanel.setBorder(new TitledBorder("Tasks"));
 		int nTasks = taskGroup.getNTasks();
@@ -885,6 +890,12 @@ public class OLProcessDialog extends PamDialog {
 	public JProgressBar getGlobalProgress() {
 		return globalProgress;
 	}
+	
+
+	public PamAlignmentPanel getTasksPanel() {
+		return tasksPanel;
+	}
+