nordic-dev.net/PAMGuard
nordic-dev.net/PAMGuard
2
0
Fork 0
mirror of https://github.com/PAMGuard/PAMGuard.git synced 2024-11-21 22:52:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Starting exporter and added new units tests for deep learning module.

Browse Source
This commit is contained in:
Jamie Mac 2024-03-15 16:55:46 +00:00
parent 6882935e9a
commit 1387428a32
36 changed files with 1470 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.settings/org.eclipse.core.resources.prefs
View File

@ -1,6 +1,5 @@
eclipse.preferences.version=1
encoding//src/rawDeepLearningClassifer/segmenter/SegmenterProcess.java=UTF-8
encoding//src/test=UTF-8
encoding//src/test/resources=UTF-8
encoding/<project>=UTF-8
encoding/src=UTF-8

6
.settings/org.eclipse.jdt.core.prefs
View File

@ -1,9 +1,9 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
org.eclipse.jdt.core.compiler.codegen.targetPlatform=21
org.eclipse.jdt.core.compiler.codegen.targetPlatform=18
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=21
org.eclipse.jdt.core.compiler.compliance=18
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
@ -13,4 +13,4 @@ org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning
org.eclipse.jdt.core.compiler.release=enabled
org.eclipse.jdt.core.compiler.source=21
org.eclipse.jdt.core.compiler.source=18

43
pom.xml
View File

@ -302,11 +302,7 @@
<version>0.0.57</version>
</dependency>
<!--jpam project - Deep learning java library
DG has commented this out since there are problems with access to the maven repo
on JDJM's Github. For now a jar file is included in the build and we'll go back to
Maven when we can ... DG 15 Jan 2022.
Repo for jpam code- this used for deep learning [15:17] Jamie MacAulay-->
<!--jpam project - Deep learning java library -->
<dependency>
<groupId>io.github.macster110</groupId>
<artifactId>jdl4pam</artifactId>
@ -372,7 +368,7 @@
<version>${javafx.version}</version>
</dependency>
<!-- Decorates JavaFX components-->
<!-- Decorates JavaFX components with error icons if inputs are incorrect-->
<dependency>
<groupId>net.synedra</groupId>
<artifactId>validatorfx</artifactId>
@ -466,6 +462,13 @@
<version>12.3.1</version>
</dependency>
<!-- Icon symbol pack for swing icons . -->
<dependency>
<groupId>org.kordamp.ikonli</groupId>
<artifactId>ikonli-swing</artifactId>
<version>12.3.1</version>
</dependency>
<!-- Ikonli MaterialDesign2 (Latest) font pack https://kordamp.org/ikonli/cheat-sheet-materialdesign2.html -->
<dependency>
<groupId>org.kordamp.ikonli</groupId>
@ -480,30 +483,28 @@
<version>12.3.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/net.sf.geographiclib/GeographicLib-Java -->
<!-- https://mvnrepository.com/artifact/net.sf.geographiclib/GeographicLib-Java
Used in the video range module-->
<dependency>
<groupId>net.sf.geographiclib</groupId>
<artifactId>GeographicLib-Java</artifactId>
<version>1.50</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.jogamp.gluegen/gluegen -->
<!-- https://mvnrepository.com/artifact/org.jogamp.gluegen/gluegen
<dependency>
<groupId>org.jogamp.gluegen</groupId>
<artifactId>gluegen-rt-main</artifactId>
<version>2.3.2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/java3d/j3dutils -->
<!-- https://mvnrepository.com/artifact/java3d/j3dcore -->
</dependency>-->
<!-- https://mvnrepository.com/artifact/com.healthmarketscience.jackcess/jackcess -->
<!-- https://mvnrepository.com/artifact/com.healthmarketscience.jackcess/jackcess
<dependency>
<groupId>com.healthmarketscience.jackcess</groupId>
<artifactId>jackcess</artifactId>
<version>3.0.1</version>
</dependency>
</dependency>-->
<!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind -->
<dependency>
@ -512,12 +513,12 @@
<version>2.10.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.jogamp.jogl/jogl-all-main -->
<!-- https://mvnrepository.com/artifact/org.jogamp.jogl/jogl-all-main
<dependency>
<groupId>org.jogamp.jogl</groupId>
<artifactId>jogl-all-main</artifactId>
<version>2.3.2</version>
</dependency>
</dependency>-->
<!-- NOT SURE IF WE NEED THE JAXB LIBRARIES HERE - NO ERRORS WHEN REMOVED
<dependency>
@ -654,7 +655,8 @@
<version>42.2.24</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.renjin/renjin-script-engine -->
<!-- https://mvnrepository.com/artifact/org.renjin/renjin-script-engine
Used to export data to R structures-->
<dependency>
<groupId>org.renjin</groupId>
<artifactId>renjin-script-engine</artifactId>
@ -669,8 +671,7 @@
<version>1.8.0-beta4</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-nop
this one is only needed to get rid of an annoying warning that shows up during startup -->
<dependency>
@ -906,7 +907,7 @@
</dependency>
<!-- https://mvnrepository.com/artifact/it.sauronsoftware/jave -->
<!-- https://mvnrepository.com/artifact/it.sauronsoftware/jave -->
<dependency>
<groupId>it.sauronsoftware</groupId>
<artifactId>jave</artifactId>

25
src/PamView/PamGui.java
View File

@ -74,8 +74,11 @@ import javax.swing.event.MenuListener;
import Acquisition.DaqSystemInterface;
import annotation.tasks.AnnotationManager;
import export.layoutFX.ExportOptions;
import metadata.MetaDataContol;
import pamViewFX.fxNodes.pamDialogFX.PamDialogFX2AWT;
import performanceTests.PerformanceDialog;
import rawDeepLearningClassifier.RawDLParams;
import tipOfTheDay.TipOfTheDayManager;
import Array.ArrayManager;
//import Logging.LogDataObserver;
@ -86,6 +89,7 @@ import PamController.PamControllerInterface;
import PamController.PamSettingManager;
import PamController.PamSettings;
import PamController.PamguardVersionInfo;
import PamController.SettingsPane;
import PamController.settings.SettingsImport;
import PamModel.CommonPluginInterface;
import PamModel.PamModel;
@ -668,6 +672,13 @@ public class PamGui extends PamView implements WindowListener, PamSettings {
startMenuEnabler.addMenuItem(menuItem);
fileMenu.add(menuItem);
//export items
menuItem = new JMenuItem("Export Data ...");
menuItem.addActionListener(new ExportOptionsListener(getGuiFrame()));
startMenuEnabler.addMenuItem(menuItem);
fileMenu.add(menuItem);
for (int i = 0; i < pamControllerInterface.getNumControlledUnits(); i++) {
menuItem = pamControllerInterface.getControlledUnit(i).createFileMenu(frame);
@ -975,6 +986,20 @@ public class PamGui extends PamView implements WindowListener, PamSettings {
}
}
class ExportOptionsListener implements ActionListener {
private JFrame parentFrame;
public ExportOptionsListener(JFrame parentFrame) {
super();
this.parentFrame = parentFrame;
}
public void actionPerformed(ActionEvent e) {
//show the export options dialog.
ExportOptions.getInstance().showDialog(parentFrame);
}
}
class MenuGeneralXMLExport implements ActionListener {
private JFrame parentFrame;
public MenuGeneralXMLExport(JFrame parentFrame) {

9
src/dataPlotsFX/TDControl.java
View File

@ -13,6 +13,7 @@ import dataPlots.TDParameters;
import dataPlotsFX.data.TDDataInfoFX;
import dataPlotsFX.layout.TDDisplayFX;
import dataPlotsFX.layout.TDGraphFX;
import detectiongrouplocaliser.DetectionGroupSummary;
/**
* The main class for the TDDisplay.
@ -222,5 +223,13 @@ public abstract class TDControl implements PamSettings {
this.uniqueDisplayName = uniqueName;
}
/**
* A new detection group has been selected.
* @param detectionGroup - the detection group that has been selected
*/
public void newSelectedDetectionGroup(DetectionGroupSummary detectionGroup, TDGraphFX tdGraph) {
}
}

8
src/dataPlotsFX/TDControlFX.java
View File

@ -12,6 +12,8 @@ import dataPlotsFX.data.TDDataInfoFX;
import dataPlotsFX.data.TDDataProviderFX;
import dataPlotsFX.data.TDDataProviderRegisterFX;
import dataPlotsFX.layout.TDDisplayFX;
import dataPlotsFX.layout.TDGraphFX;
import detectiongrouplocaliser.DetectionGroupSummary;
import PamController.PamController;
import PamguardMVC.PamDataBlock;
import PamguardMVC.PamDataUnit;
@ -394,4 +396,10 @@ public class TDControlFX extends TDControl implements UserDisplayNodeFX {
public void setFrameHolder(PamInternalPane internalFrame) {
this.internalFrame=internalFrame;
}
@Override
public void newSelectedDetectionGroup(DetectionGroupSummary detectionGroup, TDGraphFX tdGraph) {
System.out.println("New selected detection group: " + detectionGroup);
}
}

21
src/dataPlotsFX/TDDisplayController.java
View File

@ -9,10 +9,12 @@ import userDisplayFX.UserDisplayControlFX;
import userDisplayFX.UserDisplayNodeFX;
import PamController.PamController;
import PamController.PamControllerInterface;
import PamguardMVC.PamDataBlock;
import PamguardMVC.PamDataUnit;
/**
* The controller for the TD display. This is only used in the FX Gui where displays
* tend to have equality with modules.
* The controller for the TD display. This is only used in the FX GUI where displays
* tend to have equal parity with modules.
*
* @author Jamie Macaulay
*
@ -30,6 +32,8 @@ public class TDDisplayController extends UserDisplayControlFX {
*/
private TDControlFX tdControlFX;
private PamDataBlock selectedDataUnits;
public TDDisplayController(String unitName) {
@ -37,13 +41,18 @@ public class TDDisplayController extends UserDisplayControlFX {
//set the compatible data units.
//indicate that this display can accept multiple parent data blocks at the same time.
this.setMultiParent(true);
//set which PamDataUnits the display can show and therefore which datablocks it can accept as parents.
//set which PamDataUnits the display can show and therefore which data blocks it can accept as parents.
setCompatibleDataUnits();
selectedDataUnits = new PamDataBlock(PamDataUnit.class, "Selected Data Units", getUserDisplayProcess() , Integer.MAX_VALUE); //TODO
getUserDisplayProcess().addOutputDataBlock(selectedDataUnits);
}
/**
* Set compatible data units in the process for this display. This allows the data model to determine if connections can
* be made to the display.
* Set compatible data units in the process for this display. This allows the
* data model to determine if connections can be made to the display.
*/
private void setCompatibleDataUnits(){
super.removeCompatibleDataUnits();
@ -61,7 +70,6 @@ public class TDDisplayController extends UserDisplayControlFX {
* @param dataBlocks - the parent datablocks to set.
*/
protected void setParentDataBlocks(ArrayList<TDDataInfoFX> dataBlocks){
// System.out.println("Set parent datablocks: " + dataBlocks.size());
allowProcessNotify=false;
while (getUserDisplayProcess().getNumMuiltiplexDataBlocks()>0){
@ -80,6 +88,7 @@ public class TDDisplayController extends UserDisplayControlFX {
//disables notifications.
boolean allowProcessNotify=true;
@Override
public void notifyModelChanged(int type){
// System.out.println("---------------------------------" );

19
src/dataPlotsFX/overlaymark/DetectionGroupListener.java Normal file
View File

@ -0,0 +1,19 @@
package dataPlotsFX.overlaymark;
import detectiongrouplocaliser.DetectionGroupSummary;
/**
* Listener for detection group.
*
* @author Jamie Macaulay
*
*/
public interface DetectionGroupListener {
/**
* A new detection group has been selected.
* @param detectionGroup - the detection group.
*/
public void newSelectedGroup(DetectionGroupSummary detectionGroup);
}

67
src/dataPlotsFX/overlaymark/OverlayMarkerManager.java
View File

@ -42,73 +42,6 @@ public class OverlayMarkerManager extends ExtMouseAdapter {
markerAdapters.add(new DragMarkerAdapter(tdGraphFX));
}
//
// public void subscribePanel(TDPlotPanel fxPlot) {
//
// fxPlot.setOnMouseClicked(new MouseClicked());
// fxPlot.setOnMousePressed(new MousePressed());
// fxPlot.setOnMouseReleased(new MouseReleased());
// fxPlot.setOnMouseMoved(new MouseMoved());
// fxPlot.setOnMouseDragged(new MouseDragged());
// fxPlot.setOnMouseEntered(new MouseEntered());
// fxPlot.setOnMouseExited(new MouseExited());
//
// for (TDOverlayMarker tdOverlayMarker: overlayMarkers){
// tdOverlayMarker.subscribePanel(fxPlot);
// }
// }
//
// class MouseClicked implements EventHandler<MouseEvent> {
// @Override
// public void handle(MouseEvent event) {
//// System.out.println("Mouse Clicked: " + event.toString());
// overlayMarkers.get(currentMarker).mouseClicked(event);
// }
// }
//
// class MousePressed implements EventHandler<MouseEvent> {
// @Override
// public void handle(MouseEvent event) {
//// System.out.println("Mouse Pressed: " + event.toString());
// overlayMarkers.get(currentMarker).mousePressed(event);
// }
// }
// class MouseReleased implements EventHandler<MouseEvent> {
// @Override
// public void handle(MouseEvent event) {
//// System.out.println("Mouse Released: " + event.toString());
// overlayMarkers.get(currentMarker).mouseReleased(event);
// }
// }
// class MouseMoved implements EventHandler<MouseEvent> {
// @Override
// public void handle(MouseEvent event) {
// overlayMarkers.get(currentMarker).mouseMoved(event);
// }
// }
// class MouseDragged implements EventHandler<MouseEvent> {
// @Override
// public void handle(MouseEvent event) {
// overlayMarkers.get(currentMarker).mouseDragged(event);
// }
// }
//
// class MouseEntered implements EventHandler<MouseEvent> {
// @Override
// public void handle(MouseEvent event) {
//// System.out.println("Mouse Entered: " + event.toString());
// overlayMarkers.get(currentMarker).mouseEntered(event);
// }
// }
//
// class MouseExited implements EventHandler<MouseEvent> {
// @Override
// public void handle(MouseEvent event) {
//// System.out.println("Mouse Exited: " + event.toString());
// overlayMarkers.get(currentMarker).mouseExited(event);
// }
// }
public ArrayList<TDOverlayAdapter> getOverlayMarkers() {
return markerAdapters;

16
src/dataPlotsFX/overlaymark/TDMarkerAdapter.java
View File

@ -2,9 +2,15 @@ package dataPlotsFX.overlaymark;
import java.util.List;
import javax.swing.JPopupMenu;
import PamView.GeneralProjector.ParameterType;
import PamView.paneloverlay.overlaymark.MarkDataSelector;
import PamView.paneloverlay.overlaymark.MarkRelationships;
import PamView.paneloverlay.overlaymark.OverlayMark;
import PamView.paneloverlay.overlaymark.OverlayMarkObserver;
import PamView.paneloverlay.overlaymark.OverlayMarkProviders;
import PamView.paneloverlay.overlaymark.OverlayMarker;
import dataPlotsFX.layout.TDGraphFX;
import dataPlotsFX.layout.TDGraphFX.TDPlotPane;
import detectiongrouplocaliser.DetectionGroupSummary;
@ -30,7 +36,7 @@ public class TDMarkerAdapter extends TDOverlayAdapter {
/**
* The overlay marker with hooks into other modules which subscribe to the display.
*/
private StandardOverlayMarker pamMarker;
private TDMarkerFX pamMarker;
public TDMarkerAdapter(TDGraphFX tdGraphFX){
@ -51,16 +57,20 @@ public class TDMarkerAdapter extends TDOverlayAdapter {
MarkRelationships.getInstance().subscribeObservers(pamMarker);
}
pamMarker.setProjector(tdGraphFX.getGraphProjector());
pamMarker.addDetectionGroupListener((detectionGroup)->{
tdGraphFX.getTDDisplay().getTDControl().newSelectedDetectionGroup(detectionGroup, tdGraphFX);
});
}
/**
* Get the selected detections
* @return class containing selected detections
*/
public DetectionGroupSummary getSelectedDetectionGroup(){
//get the currently selected data units.
//System.out.println("Get selected detection group: " + pamMarker.getCurrentDetectionGroup());
return pamMarker.getCurrentDetectionGroup();
}

31
src/dataPlotsFX/overlaymark/TDMarkerFX.java
View File

@ -1,6 +1,7 @@
package dataPlotsFX.overlaymark;
import java.awt.Point;
import java.util.ArrayList;
import java.util.List;
import javax.swing.JPopupMenu;
@ -36,17 +37,20 @@ public class TDMarkerFX extends StandardOverlayMarker {
* The current detection group.
*/
private DetectionGroupSummary currentDetectionGroup;
/**
* The detection groups.
*/
public ArrayList<DetectionGroupListener> detectionGroupListeners = new ArrayList<DetectionGroupListener>();
/**
* The maximum distance a click can be from a data unit to be selected.
*/
private static double maxMarkDistance=15;
public TDMarkerFX(TDGraphFX tdGraphFX) {
super(tdGraphFX);
}
/*
@ -57,6 +61,23 @@ public class TDMarkerFX extends StandardOverlayMarker {
return true;
}
/**
* Add the detection group listener.
* @param detectionGroupListener - the detection group listener
*/
public void addDetectionGroupListener(DetectionGroupListener detectionGroupListener) {
detectionGroupListeners.add(detectionGroupListener);
}
/**
* Notify all listeners that there is a new detection group.
* @param detectionGroup - the detection groups.
*/
private void notifyNewDetectionGroup(DetectionGroupSummary detectionGroup) {
for (DetectionGroupListener aDetectionGroupListener : detectionGroupListeners) {
aDetectionGroupListener.newSelectedGroup(detectionGroup);
}
}
/**
* Override to allow a highlighting of selected data units.
@ -110,7 +131,7 @@ public class TDMarkerFX extends StandardOverlayMarker {
currentDetectionGroup = new DetectionGroupSummary(e, this, this.getCurrentMark(),
getSelectedMarkedDataUnits(this.getCurrentMark(), null) );
notifyNewDetectionGroup(currentDetectionGroup);
// Debug.out.println("TDMarkerFX: Marked data units: " + currentDetectionGroup.getNumDataUnits());
this.setNowMarking(false);
@ -160,6 +181,7 @@ public class TDMarkerFX extends StandardOverlayMarker {
//System.out.println("PamMarkerFX: Found a data unit @ distance: " + foundDataUnit.distance + " "
// + PamCalendar.formatDateTime(foundDataUnit.dataUnit.getTimeMilliseconds()));
currentDetectionGroup = new DetectionGroupSummary(e, this, this.getCurrentMark(), foundDataUnit.dataUnit);
notifyNewDetectionGroup(currentDetectionGroup);
return true;
}
else{
@ -168,6 +190,7 @@ public class TDMarkerFX extends StandardOverlayMarker {
//only do this if the primary mouse button. Otherwise we might be right clicking on a markout out area in which
//all saved markes will be removed and the pop up menu will show nothing. So the user can only use the primary button to select data units.
currentDetectionGroup =null;
notifyNewDetectionGroup(currentDetectionGroup);
}
foundDataUnit=null;
return false;

2
src/dataPlotsFX/overlaymark/TDOverlayAdapter.java
View File

@ -9,7 +9,7 @@ import javafx.scene.canvas.GraphicsContext;
import javafx.scene.control.Tooltip;
/**
* An overlay adapter essentially allows user mouse and touch interactions with the display.
* An overlay adapter allows user mouse and touch interactions with the display.
*
* @author Jamie Macaulay
*

36
src/export/MLExport/MLDetectionsManager.java
View File

@ -1,12 +1,11 @@
package export.MLExport;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import com.google.protobuf.compiler.PluginProtos.CodeGeneratorResponse.File;
import PamguardMVC.PamDataUnit;
import export.PamExportManager;
import export.PamDataUnitExporter;
import us.hebi.matlab.mat.format.Mat5;
import us.hebi.matlab.mat.types.Struct;
@ -16,7 +15,7 @@ import us.hebi.matlab.mat.types.Struct;
* @author Jamie Macaulay
*
*/
public class MLDetectionsManager implements PamExportManager {
public class MLDetectionsManager implements PamDataUnitExporter {
public static final String extension = "mat";
@ -35,7 +34,15 @@ public class MLDetectionsManager implements PamExportManager {
@Override
public boolean hasCompatibleUnits(Class dataUnitType) {
// TODO Auto-generated method stub
for (int i=0; i<mlDataUnitsExport.size(); i++){
//check whether the same. ;
//System.out.println(" dataUnits.get(j).getClass(): " + dataUnits.get(j).getClass());
//System.out.println(" mlDataUnitsExport.get(i).getUnitClass(): " + mlDataUnitsExport.get(i).getUnitClass());
if (mlDataUnitsExport.get(i).getUnitClass().isAssignableFrom(dataUnitType)) {
//System.out.println("FOUND THE DATA UNIT!");
return true;
}
}
return false;
}
@ -52,15 +59,7 @@ public class MLDetectionsManager implements PamExportManager {
public boolean hasCompatibleUnits(List<PamDataUnit> dataUnits) {
//first need to figure out how many data units there are.
for (int j=0; j<dataUnits.size(); j++){
for (int i=0; i<mlDataUnitsExport.size(); i++){
//check whether the same. ;
//System.out.println(" dataUnits.get(j).getClass(): " + dataUnits.get(j).getClass());
//System.out.println(" mlDataUnitsExport.get(i).getUnitClass(): " + mlDataUnitsExport.get(i).getUnitClass());
if (mlDataUnitsExport.get(i).getUnitClass().isAssignableFrom(dataUnits.get(j).getClass())) {
//System.out.println("FOUND THE DATA UNIT!");
return true;
}
}
if (hasCompatibleUnits(dataUnits.get(j).getClass())) return true;
}
return false;
}
@ -115,16 +114,12 @@ public class MLDetectionsManager implements PamExportManager {
n++;
alreadyStruct[j] = true;
}
}
if (n>=1) {
list.set(mlDataUnitsExport.get(i).getName(),mlStructure);
list.set(mlDataUnitsExport.get(i).getName()+"_sR", Mat5.newScalar(sampleRate));
}
}
//now ready to save.
@ -137,6 +132,11 @@ public class MLDetectionsManager implements PamExportManager {
return extension;
}
@Override
public String getIconString() {
return "file-matlab";
}

13
src/export/PamExportManager.java → src/export/PamDataUnitExporter.java
View File

@ -1,15 +1,15 @@
package export;
import java.io.File;
import java.util.List;
import com.google.protobuf.compiler.PluginProtos.CodeGeneratorResponse.File;
import PamguardMVC.PamDataUnit;
/**
* Manages the exporting of data units to a particular type of file e.g. CSV, MATLAB, R, WAV files
*/
public interface PamExportManager {
public interface PamDataUnitExporter {
/**
* Check whether a particular data unit class is compatible
@ -31,6 +31,13 @@ public interface PamExportManager {
* Get the extension for the output file type
* @return the extension for the file type e.g. "mat"
*/
public String getFileExtension();
public String getFileExtension();
/**
* Get the ikonli icon string for the exporter.
* @return the ikon string.
*/
public String getIconString();
}

26
src/export/PamExporter.java
View File

@ -1,26 +0,0 @@
package export;
import java.util.ArrayList;
import export.MLExport.MLDetectionsManager;
/**
* Exports data to external files.
*/
public class PamExporter {
/**
* A list of the avilable exporters.
*/
private ArrayList<PamExportManager> pamExporters;
public PamExporter() {
pamExporters = new ArrayList<PamExportManager>();
//add the MATLAB export
pamExporters.add(new MLDetectionsManager());
}
}

69
src/export/PamExporterManager.java Normal file
View File

@ -0,0 +1,69 @@
package export;
import java.util.ArrayList;
import PamguardMVC.PamDataBlock;
import PamguardMVC.PamDataUnit;
import export.MLExport.MLDetectionsManager;
import export.RExport.RExportManager;
import export.wavExport.WavFileExportManager;
/**
* Exports data to external files. Manages the file sizes and creates data buffers for
* offline processing.
*/
public class PamExporterManager {
/**
* A list of the avilable exporters.
*/
private ArrayList<PamDataUnitExporter> pamExporters;
public PamExporterManager() {
pamExporters = new ArrayList<PamDataUnitExporter>();
//add the MATLAB export
pamExporters.add(new MLDetectionsManager());
pamExporters.add(new RExportManager());
pamExporters.add(new WavFileExportManager());
}
/**
* Add a data unit to the export list.
*/
public void exportDataUnit(PamDataUnit<?, ?> dataUnit) {
// TODO Auto-generated method stub
}
public boolean canExportDataBlock(PamDataBlock dataBlock) {
for (PamDataUnitExporter exporter:pamExporters) {
if (exporter.hasCompatibleUnits(dataBlock.getUnitClass())) return true;
}
return false;
}
/**
* Get the number of data exporters. e.g. MAT files, CSV, Wav etc.
* @return the number of data exporters
*/
public int getNumExporters() {
return pamExporters.size();
}
/**
* Get a specific exporter.
* @param i - the index of the exporter.
* @return the exporter.
*/
public PamDataUnitExporter getExporter(int i) {
return pamExporters.get(i);
}
}

54
src/export/RExport/RExportManager.java
View File

@ -1,11 +1,14 @@
package export.RExport;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import org.renjin.sexp.ListVector;
import org.renjin.sexp.PairList;
import PamguardMVC.PamDataUnit;
import export.PamDataUnitExporter;
/**
* Handles exporting pam data units into an rdata.
@ -13,7 +16,7 @@ import PamguardMVC.PamDataUnit;
*
*/
@SuppressWarnings("rawtypes")
public class RExportManager {
public class RExportManager implements PamDataUnitExporter {
/**
*
@ -27,29 +30,34 @@ public class RExportManager {
mlDataUnitsExport.add(new RClickExport());
mlDataUnitsExport.add(new RWhistleExport());
mlDataUnitsExport.add(new RRawExport()); //should be last in case raw data holders have specific exporters
}
/**
* Check whether there are compatible data units to be exported.
* @param dataUnits - the data unit list
* @return true if MATLAB export is possible for the current data units.
* @return true if r export is possible for the current data units.
*/
public boolean hasCompatibleUnits(List<PamDataUnit> dataUnits) {
for (int i=0; i<mlDataUnitsExport.size(); i++){
//first need to figure out how many data units there are.
for (int j=0; j<dataUnits.size(); j++){
//check whether the same. ;
// System.out.println(" dataUnits.get(j).getClass(): " + dataUnits.get(j).getClass());
// System.out.println(" mlDataUnitsExport.get(i).getUnitClass(): " + mlDataUnitsExport.get(i).getUnitClass());
if (mlDataUnitsExport.get(i).getUnitClass().isAssignableFrom(dataUnits.get(j).getClass())) {
return true;
}
}
//first need to figure out how many data units there are.
for (int j=0; j<dataUnits.size(); j++){
if (hasCompatibleUnits(dataUnits.get(j).getClass())) return true;
}
return false;
}
@Override
public boolean hasCompatibleUnits(Class dataUnitType) {
for (int i=0; i<mlDataUnitsExport.size(); i++){
//check whether the same. ;
//System.out.println(" dataUnits.get(j).getClass(): " + dataUnits.get(j).getClass());
//System.out.println(" mlDataUnitsExport.get(i).getUnitClass(): " + mlDataUnitsExport.get(i).getUnitClass());
if (mlDataUnitsExport.get(i).getUnitClass().isAssignableFrom(dataUnitType)) {
//System.out.println("FOUND THE DATA UNIT!");
return true;
}
}
return false;
}
/**
* Sort a list of data units into lists of the same type of units. Convert to a list of structures.
@ -126,6 +134,24 @@ public class RExportManager {
public ArrayList<String> dataUnitTypes;
}
@Override
public boolean exportData(File fileName, List<PamDataUnit> dataUnits) {
// TODO Auto-generated method stub
return false;
}
@Override
public String getFileExtension() {
return ".RData";
}
@Override
public String getIconString() {
return "file-r";
}
}

113
src/export/layoutFX/ExportOptions.java Normal file
View File

@ -0,0 +1,113 @@
package export.layoutFX;
import java.io.Serializable;
import javax.swing.JFrame;
import PamController.PamControlledUnitSettings;
import PamController.PamSettingManager;
import PamController.PamSettings;
import PamController.StorageParameters;
import export.PamExporterManager;
import export.swing.ExportProcessDialog;
/**
* Manages opening settings dialog and saving settings.
*
* @author Jamie Macaulay
*
*/
public class ExportOptions implements PamSettings {
private static ExportOptions singleInstance;
/**
* Parameters for the exporter.
*/
private ExportParams storageParameters = new ExportParams();
/**
* Swing dialog for exporting data.
*/
private ExportProcessDialog exportProcessDialog;
/**
* Reference to the export manager. This handles exporting data units.
*/
private PamExporterManager exportManager = new PamExporterManager();
private ExportOptions() {
PamSettingManager.getInstance().registerSettings(this);
}
public static ExportOptions getInstance() {
if (singleInstance == null) {
singleInstance = new ExportOptions();
}
return singleInstance;
}
/**
* Show the swing dialog.
* @param parentFrame - the parent frame.
* @return true if settings are OK on close
*/
public boolean showDialog(JFrame parentFrame) {
if (exportProcessDialog==null) {
exportProcessDialog= new ExportProcessDialog(exportManager);
}
this.exportProcessDialog.showOfflineDialog(parentFrame);
// ExportParams newParams = StorageOptionsDialog.showDialog(parentFrame, storageParameters);
// if (newParams != null) {
// storageParameters = newParams.clone();
// return true;
// }
// else {
// return false;
// }
return true;
}
@Override
public Serializable getSettingsReference() {
return storageParameters;
}
@Override
public long getSettingsVersion() {
return StorageParameters.serialVersionUID;
}
@Override
public String getUnitName() {
return "PAMGUARD Export Options";
}
@Override
public String getUnitType() {
return "PAMGUARD Export Options";
}
@Override
public boolean restoreSettings(
PamControlledUnitSettings pamControlledUnitSettings) {
storageParameters = ((ExportParams) pamControlledUnitSettings.getSettings()).clone();
return true;
}
public void setStorageParameters(ExportParams storageParameters) {
this.storageParameters = storageParameters;
}
/**
* Get storage parameters settings.
* @return the storage paramters settings
*/
public ExportParams getStorageParameters() {
return storageParameters;
}
}

32
src/export/layoutFX/ExportParams.java Normal file
View File

@ -0,0 +1,32 @@
package export.layoutFX;
import java.io.Serializable;
/**
* Parameter for the exporter.
* @author Jamie Macaulay
*
*/
public class ExportParams implements Serializable, Cloneable {
/**
*
*/
private static final long serialVersionUID = 1L;
public int exportChoice;
@Override
public ExportParams clone() {
try {
ExportParams newP = (ExportParams) super.clone();
return newP;
} catch (CloneNotSupportedException e) {
e.printStackTrace();
return null;
}
}
}

53
src/export/layoutFX/ExporterPane.java Normal file
View File

@ -0,0 +1,53 @@
package export.layoutFX;
import PamController.SettingsPane;
import javafx.scene.Node;
import javafx.scene.control.Label;
import pamViewFX.fxNodes.PamBorderPane;
/**
* Create a pane to export data.
* @author Jamie Macaulay
*
*/
public class ExporterPane extends SettingsPane<ExportParams>{
PamBorderPane mainPane;
public ExporterPane(Object ownerWindow) {
super(ownerWindow);
mainPane = new PamBorderPane();
mainPane.setTop(new Label("Hello"));
mainPane.setCenter(new Label("Hello"));
}
@Override
public ExportParams getParams(ExportParams currParams) {
// TODO Auto-generated method stub
return null;
}
@Override
public void setParams(ExportParams input) {
// TODO Auto-generated method stub
}
@Override
public String getName() {
return "Export Settings";
}
@Override
public Node getContentNode() {
return mainPane;
}
@Override
public void paneInitialized() {
// TODO Auto-generated method stub
}
}

177
src/export/swing/ExportProcessDialog.java Normal file
View File

@ -0,0 +1,177 @@
package export.swing;
import java.awt.BorderLayout;
import java.awt.Color;
import java.awt.Frame;
import java.awt.Window;
import java.awt.event.ActionListener;
import java.util.ArrayList;
import javax.swing.ButtonGroup;
import javax.swing.Icon;
import javax.swing.JToggleButton;
import org.kordamp.ikonli.Ikon;
import org.kordamp.ikonli.fileicons.FileIcons;
import org.kordamp.ikonli.materialdesign2.MaterialDesignA;
import org.kordamp.ikonli.materialdesign2.MaterialDesignF;
import org.kordamp.ikonli.swing.FontIcon;
import PamController.PamController;
import PamView.panel.PamPanel;
import PamguardMVC.PamDataBlock;
import export.PamExporterManager;
import offlineProcessing.OLProcessDialog;
import offlineProcessing.OfflineTaskGroup;
/**
* Processes exportying data.
* @author Jamie Macaulay
*
*/
public class ExportProcessDialog {
/**
* The offline task group
*/
private OfflineTaskGroup dlOfflineGroup;
private OLProcessDialog mtOfflineDialog;
/**
* Reference to the export manager.
*/
private PamExporterManager exportManager;
public ExportProcessDialog(PamExporterManager exportManager) {
//create the offline task group.
this.exportManager=exportManager;
dlOfflineGroup = new ExportTaskGroup("Export data");
}
public void createExportGroup() {
//clear current tasks.
dlOfflineGroup.clearTasks();
//go through every data block we have and check if we can export the data units...
ArrayList<PamDataBlock> dataBlocks= PamController.getInstance().getDataBlocks();
for (int i=0; i<dataBlocks.size(); i++) {
if (exportManager.canExportDataBlock(dataBlocks.get(i))) {
dlOfflineGroup.addTask(new ExportTask(dataBlocks.get(i), exportManager));
}
}
}
////---Swing stuff----/// should not be here but this is how PG works.
public void showOfflineDialog(Frame parentFrame) {
createExportGroup();
//if null open the dialog- also create a new offlineTask group if the datablock has changed.
if (mtOfflineDialog == null) {
mtOfflineDialog = new ExportOLDialog(parentFrame,
dlOfflineGroup, "Export Data");
//batchLocaliseDialog.setModalityType(Dialog.ModalityType.MODELESS);
}
mtOfflineDialog.enableControls();
mtOfflineDialog.setVisible(true);
}
/**
* Custom dialog which shows some extra options/
* @author Jamie Macaulay
*
*/
class ExportOLDialog extends OLProcessDialog {
/**
*
*/
private static final long serialVersionUID = 1L;
public ExportOLDialog(Window parentFrame, OfflineTaskGroup taskGroup, String title) {
super(parentFrame, taskGroup, title);
// TODO Auto-generated constructor stub
BorderLayout layout = new BorderLayout();
PamPanel mainPanel = new PamPanel();
ButtonGroup buttonGroup = new ButtonGroup();
PamPanel buttonPanel = new PamPanel();
ActionListener listener = actionEvent -> {
System.out.println(actionEvent.getActionCommand() + " Selected");
};
for (int i = 0; i < exportManager.getNumExporters(); i++) {
JToggleButton b = new JToggleButton();
FontIcon icon = FontIcon.of(getIconFromString(exportManager.getExporter(i).getIconString()));
icon.setIconSize(25);
icon.setIconColor(Color.GRAY);
b.setIcon(icon);
b.addActionListener(listener);
buttonGroup.add(b);
buttonPanel.add(b);
}
mainPanel.add(buttonPanel, BorderLayout.CENTER);
//add the main panel at a diffderent index.
getMainPanel().add(buttonPanel, 1);
}
private Ikon getIconFromString(String iconString) {
Ikon icon = null;
/**
* This is nasty but we won't have many exporters and this is the only
* good way to get this to work in Swing.
*/
switch (iconString) {
case "file-matlab":
icon=FileIcons.MATLAB;
break;
case "file-r":
icon=FileIcons.R;
break;
case "mdi2f-file-music":
icon=MaterialDesignF.FILE_MUSIC;
break;
}
return icon;
}
}
class ExportTaskGroup extends OfflineTaskGroup{
public ExportTaskGroup(String settingsName) {
super(null, settingsName);
// TODO Auto-generated constructor stub
}
@Override
public String getUnitType() {
return "Export Data";
}
}
}

48
src/export/swing/ExportTask.java Normal file
View File

@ -0,0 +1,48 @@
package export.swing;
import PamguardMVC.PamDataBlock;
import PamguardMVC.PamDataUnit;
import dataMap.OfflineDataMapPoint;
import export.PamExporterManager;
import offlineProcessing.OfflineTask;
/**
* Export data to a file type.
*
* @author Jamie Macaulay
*
*/
public class ExportTask extends OfflineTask<PamDataUnit<?,?>>{
PamExporterManager exporter;
public ExportTask(PamDataBlock<PamDataUnit<?, ?>> parentDataBlock, PamExporterManager exporter) {
super(parentDataBlock);
this.exporter = exporter;
// TODO Auto-generated constructor stub
}
@Override
public String getName() {
return this.getDataBlock().getDataName();
}
@Override
public boolean processDataUnit(PamDataUnit<?, ?> dataUnit) {
exporter.exportDataUnit(dataUnit);
return true;
}
@Override
public void newDataLoad(long startTime, long endTime, OfflineDataMapPoint mapPoint) {
// TODO Auto-generated method stub
}
@Override
public void loadedDataComplete() {
// TODO Auto-generated method stub
}
}

35
src/export/wavExport/WavFileExportManager.java
View File

@ -22,6 +22,7 @@ import PamguardMVC.PamRawDataBlock;
import PamguardMVC.dataOffline.OfflineDataLoading;
import dataMap.OfflineDataMapPoint;
import detectiongrouplocaliser.DetectionGroupSummary;
import export.PamDataUnitExporter;
import wavFiles.Wav16AudioFormat;
import wavFiles.WavFileWriter;
@ -30,7 +31,7 @@ import wavFiles.WavFileWriter;
* @author Jamie Macaulay
*
*/
public class WavFileExportManager {
public class WavFileExportManager implements PamDataUnitExporter {
/**
* Successful writing of .wav file.
@ -420,6 +421,38 @@ public class WavFileExportManager {
@Override
public boolean hasCompatibleUnits(Class dataUnitType) {
// TODO Auto-generated method stub
return false;
}
@Override
public boolean exportData(File fileName,
List<PamDataUnit> dataUnits) {
// TODO Auto-generated method stub
return false;
}
@Override
public String getFileExtension() {
return "wav";
}
@Override
public String getIconString() {
// TODO Auto-generated method stub
return "mdi2f-file-music";
}
// hello(){

15
src/offlineProcessing/OLProcessDialog.java
View File

@ -93,13 +93,18 @@ public class OLProcessDialog extends PamDialog {
TaskStatus currentStatus = TaskStatus.IDLE;
/**
* Reference to the main panel
*/
private JPanel mainPanel;
public OLProcessDialog(Window parentFrame, OfflineTaskGroup taskGroup, String title) {
super(parentFrame, title, false);
this.taskGroup = taskGroup;
taskGroup.setTaskMonitor(new OLMonitor());
JPanel mainPanel = new JPanel();
mainPanel = new JPanel();
mainPanel.setLayout(new BoxLayout(mainPanel, BoxLayout.Y_AXIS));
JPanel dataSelectPanel = new PamAlignmentPanel(BorderLayout.WEST);
@ -207,6 +212,14 @@ public class OLProcessDialog extends PamDialog {
setResizable(true);
}
/**
* Get the main panel. This can be used to add additional controls if needed.
* @return the main panel.
*/
public JPanel getMainPanel() {
return mainPanel;
}
/**
* Get the delete old data check box

13
src/offlineProcessing/OfflineTaskGroup.java
View File

@ -66,6 +66,7 @@ public class OfflineTaskGroup implements PamSettings {
private DataTimeLimits dataTimeLimits;
private volatile TaskStatus completionStatus = TaskStatus.IDLE;
/**
* PamControlledunit required in constructor since some bookkeeping will
* be going on in the background which will need the unit type and name.
@ -76,7 +77,7 @@ public class OfflineTaskGroup implements PamSettings {
public OfflineTaskGroup(PamControlledUnit pamControlledUnit, String settingsName) {
super();
this.pamControlledUnit = pamControlledUnit;
pamControlledUnit.addOfflineTaskGroup(this);
if (pamControlledUnit!=null) pamControlledUnit.addOfflineTaskGroup(this);
this.settingsName = settingsName;
PamSettingManager.getInstance().registerSettings(this);
}
@ -965,4 +966,14 @@ public class OfflineTaskGroup implements PamSettings {
public OfflineSuperDetFilter getSuperDetectionFilter() {
return superDetectionFilter;
}
/**
* Clear all task from the task group. This also clears affected and required datablocks.
*/
public void clearTasks() {
requiredDataBlocks.clear();
affectedDataBlocks.clear();
offlineTasks.clear();
}
}

35
src/rawDeepLearningClassifier/defaultModels/HumpbackWhaleAtlantic.java
View File

@ -5,8 +5,17 @@ import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import org.jamdev.jdl4pam.transforms.DLTransformsFactory;
import org.jamdev.jdl4pam.transforms.DLTransfromParams;
import org.jamdev.jdl4pam.transforms.SimpleTransformParams;
import org.jamdev.jdl4pam.transforms.DLTransform.DLTransformType;
import rawDeepLearningClassifier.dlClassification.DLClassName;
import rawDeepLearningClassifier.dlClassification.animalSpot.StandardModelParams;
import rawDeepLearningClassifier.dlClassification.genericModel.GenericModelParams;
import rawDeepLearningClassifier.layoutFX.exampleSounds.ExampleSoundFactory.ExampleSoundType;
/**
*
@ -57,7 +66,31 @@ public class HumpbackWhaleAtlantic implements DLModel {
@Override
public void setParams(Serializable dlModelSettings) {
// TODO Auto-generated method stub
GenericModelParams genericModelParams = (GenericModelParams) dlModelSettings;
//decimation value
float sr = 2000;
//create the transforms.
ArrayList<DLTransfromParams> dlTransformParamsArr = new ArrayList<DLTransfromParams>();
dlTransformParamsArr.add(new SimpleTransformParams(DLTransformType.DECIMATE_SCIPY, sr));
genericModelParams.dlTransfromParams = dlTransformParamsArr;
genericModelParams.defaultSegmentLen = 3.8775*1000;
genericModelParams.binaryClassification = new boolean[] {true};
genericModelParams.classNames= new DLClassName[] {new DLClassName("Humpback whale", (short) 1)};
genericModelParams.numClasses = 1;
genericModelParams.defaultShape= new Long[] {-1L,-1L,-1L,1L};
genericModelParams.shape= new Long[] {-1L,-1L,-1L,1L};
//create the transforms.
genericModelParams.dlTransfroms = DLTransformsFactory.makeDLTransforms((ArrayList<DLTransfromParams>)genericModelParams.dlTransfromParams);
genericModelParams.setExampleSound(ExampleSoundType.HUMPBACK_WHALE);
}

10
src/rawDeepLearningClassifier/defaultModels/HumpbackWhaleGoogle.java
View File

@ -36,8 +36,8 @@ public class HumpbackWhaleGoogle implements DLModel {
@Override
public URI getModelURI() {
try {
return new URL("https://github.com/PAMGuard/deeplearningmodels/raw/master/humpback_whale_1/humpback_whale_1.zip").toURI();
} catch (MalformedURLException | URISyntaxException e) {
return new URI("https://github.com/PAMGuard/deeplearningmodels/raw/master/humpback_whale_1/humpback_whale_1.zip");
} catch (URISyntaxException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
@ -52,8 +52,8 @@ public class HumpbackWhaleGoogle implements DLModel {
@Override
public URI getCitationLink() {
try {
return new URL("https://doi.org/10.3389/fmars.2021.607321").toURI();
} catch (MalformedURLException | URISyntaxException e) {
return new URI("https://doi.org/10.3389/fmars.2021.607321");
} catch (URISyntaxException e) {
// TODO Auto-generated catch block
e.printStackTrace();
return null;
@ -70,7 +70,7 @@ public class HumpbackWhaleGoogle implements DLModel {
//create the transforms.
ArrayList<DLTransfromParams> dlTransformParamsArr = new ArrayList<DLTransfromParams>();
dlTransformParamsArr.add(new SimpleTransformParams(DLTransformType.DECIMATE, sr));
dlTransformParamsArr.add(new SimpleTransformParams(DLTransformType.DECIMATE_SCIPY, sr));
genericModelParams.dlTransfromParams = dlTransformParamsArr;

4
src/rawDeepLearningClassifier/defaultModels/RightWhaleModel1.java
View File

@ -44,8 +44,8 @@ public class RightWhaleModel1 implements DLModel {
@Override
public URI getModelURI() {
try {
return new URL("https://github.com/PAMGuard/deeplearningmodels/raw/master/right_whale_1/model_lenet_dropout_input_conv_all.zip").toURI();
} catch (MalformedURLException | URISyntaxException e) {
return new URI("https://github.com/PAMGuard/deeplearningmodels/raw/master/right_whale_1/model_lenet_dropout_input_conv_all.zip");
} catch (URISyntaxException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}

2
src/rawDeepLearningClassifier/dlClassification/genericModel/GenericModelWorker.java
View File

@ -44,7 +44,7 @@ public class GenericModelWorker extends DLModelWorker<GenericPrediction> {
waveStack[i] = transformedDataStack[i][0];
}
//System.out.println("RUN GENERIC MODEL WAVE: " + transformedDataStack.length + " " + transformedDataStack[0].length);
//System.out.println("RUN GENERIC MODEL WAVE: " + waveStack.length + " " + waveStack[0].length + " " + waveStack[0][0]);
results = getModel().runModel(waveStack);
}
//System.out.println("GENERIC MODEL RESULTS: " + results== null ? null : results.length);

115
src/test/rawDeepLearningClassifier/GenericDLClassifierTest.java
View File

@ -3,6 +3,10 @@ package test.rawDeepLearningClassifier;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
@ -14,6 +18,7 @@ import org.jamdev.jdl4pam.utils.DLUtils;
import org.jamdev.jpamutils.wavFiles.AudioData;
import org.junit.jupiter.api.Test;
import rawDeepLearningClassifier.defaultModels.HumpbackWhaleAtlantic;
import rawDeepLearningClassifier.defaultModels.RightWhaleModel1;
import rawDeepLearningClassifier.dlClassification.genericModel.GenericModelParams;
import rawDeepLearningClassifier.dlClassification.genericModel.GenericModelWorker;
@ -29,10 +34,14 @@ import rawDeepLearningClassifier.segmenter.SegmenterProcess.GroupedRawData;
public class GenericDLClassifierTest {
/**
* Run a test on the Generic DL Classifier. This tests the worker can open and run a model
* Run a test on the Generic DL Classifier. This tests the worker can open and
* run a model
*/
@Test
public void rightWhaleDLWorker() {
System.out.println("*****Generic DL: Right whale test*****");
//relative paths to the resource folders.
String relModelPath = "./src/test/resources/rawDeepLearningClassifier/Generic/right_whale/model_lenet_dropout_input_conv_all/saved_model.pb";
String relWavPath = "./src/test/resources/rawDeepLearningClassifier/Generic/right_whale/right_whale_example.wav";
@ -63,18 +72,18 @@ public class GenericDLClassifierTest {
GroupedRawData groupedRawData = new GroupedRawData(0, 1, 0, duration, (int) duration);
groupedRawData.copyRawData(soundData.getScaledSampleAmplitudes(), 0, soundData.getScaledSampleAmplitudes().length, 0);
ArrayList<GroupedRawData> groupedData = new ArrayList<GroupedRawData>();
groupedData.add(groupedRawData);
ArrayList<GenericPrediction> gwenericPrediciton = genericModelWorker.runModel(groupedData, soundData.sampleRate, 0);
float[] output = gwenericPrediciton.get(0).getPrediction();
System.out.println("Right whale network output: " + output[0] + " " + output[1]);
//test the predicitons are true.
assertTrue( output[0]<0.01 ); //noise
assertTrue( output[1]>0.98 ); //right whale
@ -87,18 +96,98 @@ public class GenericDLClassifierTest {
assertEquals(false, true);
}
}
/**
* Test the ketos classifier and tests are working properly. This tests loading the ketos model and also using
* functions in KetosWorker.
* Test the ketos classifier and tests are working properly. This tests loading
* the ketos model and also using functions in KetosWorker.
*/
@Test
public void humpbackWhaleTest() {
System.out.println("*****Generic DL: Humpback whale test*****");
//relative paths to the resource folders.
String relModelPath = "./src/test/resources/rawDeepLearningClassifier/Generic/humpback_whale_atlantic/FlatHBNA/saved_model.pb";
String relWavPath = "./src/test/resources/rawDeepLearningClassifier/Generic/humpback_whale_atlantic/SAMOSAS_EL1_humpback.wav";
String resultsPath = "./src/test/resources/rawDeepLearningClassifier/Generic/humpback_whale_atlantic/SAMOSAS_EL1_humpback_annotations.txt";
Path path = Paths.get(relModelPath);
GenericModelWorker genericModelWorker = new GenericModelWorker();
GenericModelParams genericModelParams = new GenericModelParams();
genericModelParams.modelPath = path.toAbsolutePath().normalize().toString();
path = Paths.get(relWavPath);
String wavFilePath = path.toAbsolutePath().normalize().toString();
HumpbackWhaleAtlantic humpbackModel = new HumpbackWhaleAtlantic();
humpbackModel.setParams(genericModelParams);
System.out.println(genericModelParams);
//prep the model
genericModelWorker.prepModel(genericModelParams, null);
try {
//load audio
AudioData soundData = DLUtils.loadWavFile(wavFilePath);
//load true predictions file.
File file = new File(resultsPath);
FileReader fr = new FileReader(file);
BufferedReader br = new BufferedReader(fr);
String line;
int ind=0;
while((line = br.readLine()) != null){
if (ind>0) {
//read the data from the text file
String[] data = line.split("\t");
int chunkID = Integer.valueOf(data[0]);
double startTimeS = Double.valueOf(data[1]);
double endTimeS = Double.valueOf(data[2]);
double prediction = Double.valueOf(data[5]);
int startChunk = (int) (soundData.sampleRate*startTimeS); //humpback whale call
int chunkSize = (int) Math.ceil((endTimeS-startTimeS)*soundData.sampleRate);
chunkSize=7755;
GroupedRawData groupedRawData = new GroupedRawData(0, 1, 0, chunkSize, chunkSize);
groupedRawData.copyRawData(soundData.getScaledSampleAmplitudes(), startChunk, chunkSize, 0);
ArrayList<GroupedRawData> groupedData = new ArrayList<GroupedRawData>();
groupedData.add(groupedRawData);
ArrayList<GenericPrediction> genericPrediction = genericModelWorker.runModel(groupedData, soundData.sampleRate, 0);
float[] output = genericPrediction.get(0).getPrediction();
System.out.println(String.format("Chunk %d %d Predicted output: %.2f true output: %.2f passed: %b", chunkID, startChunk,
output[0], prediction, output[0]>prediction*0.9 && output[0]<prediction*1.1));
//allow 10% scrumph to take account of slight differences in Java input.
assertEquals(output[0], prediction, 0.1); //humpback whale detection
}
ind++;
}
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}

348
src/test/resources/rawDeepLearningClassifier/Generic/humpback_whale_atlantic/FlatHBNA/keras_metadata.pb Normal file
View File

File diff suppressed because one or more lines are too long

BIN
src/test/resources/rawDeepLearningClassifier/Generic/humpback_whale_atlantic/FlatHBNA/saved_model.pb Normal file
View File

Binary file not shown.

BIN
src/test/resources/rawDeepLearningClassifier/Generic/humpback_whale_atlantic/FlatHBNA/variables/variables.data-00000-of-00001 Normal file
View File

Binary file not shown.

BIN
src/test/resources/rawDeepLearningClassifier/Generic/humpback_whale_atlantic/FlatHBNA/variables/variables.index Normal file
View File

Binary file not shown.

BIN
src/test/resources/rawDeepLearningClassifier/Generic/humpback_whale_atlantic/SAMOSAS_EL1_humpback.wav Normal file
View File

Binary file not shown.

216
src/test/resources/rawDeepLearningClassifier/Generic/humpback_whale_atlantic/SAMOSAS_EL1_humpback_annotations.txt Normal file
View File

@ -0,0 +1,216 @@
Selection Begin Time (s) End Time (s) High Freq (Hz) Low Freq (Hz) Prediction/Comments
1 3.8775 7.755 1000 50 0.99972177
2 7.755 11.6325 1000 50 0.89199716
3 11.6325 15.51 1000 50 0.64813995
4 15.51 19.3875 1000 50 0.9713266
5 19.3875 23.265 1000 50 0.8342721
6 23.265 27.142500000000002 1000 50 0.99634343
7 31.02 34.8975 1000 50 0.9933128
8 34.8975 38.775 1000 50 0.5325079
9 42.6525 46.53 1000 50 0.9727334
10 46.53 50.4075 1000 50 0.7700455
11 54.285 58.162499999999994 1000 50 0.93229276
12 58.1625 62.04 1000 50 0.5089565
13 62.04 65.9175 1000 50 0.53604007
14 65.9175 69.795 1000 50 0.6665228
15 69.795 73.6725 1000 50 0.8109671
16 73.6725 77.55 1000 50 0.9998325
17 77.55 81.4275 1000 50 0.6296093
18 81.4275 85.30499999999999 1000 50 0.99747694
19 93.06 96.9375 1000 50 0.99976456
20 96.9375 100.815 1000 50 0.53218305
21 104.6925 108.57 1000 50 0.98100567
22 112.4475 116.325 1000 50 0.91926867
23 124.08 127.9575 1000 50 0.9990459
24 127.9575 131.835 1000 50 0.6861532
25 135.7125 139.59 1000 50 0.5737106
26 143.4675 147.345 1000 50 0.9537415
27 147.345 151.2225 1000 50 0.5592457
28 151.2225 155.1 1000 50 0.97674406
29 155.1 158.9775 1000 50 0.55292696
30 166.7325 170.60999999999999 1000 50 0.86637044
31 170.61 174.4875 1000 50 0.8887141
32 174.4875 178.365 1000 50 0.65914917
33 182.2425 186.12 1000 50 0.78331536
34 189.9975 193.875 1000 50 0.51390845
35 193.875 197.7525 1000 50 0.8545577
36 197.7525 201.63 1000 50 0.99297434
37 209.385 213.2625 1000 50 0.91762686
38 217.14 221.01749999999998 1000 50 0.9865262
39 221.0175 224.895 1000 50 0.9802332
40 224.895 228.7725 1000 50 0.9038371
41 228.7725 232.65 1000 50 0.9299754
42 232.65 236.5275 1000 50 0.99995977
43 236.5275 240.405 1000 50 0.75877947
44 240.405 244.2825 1000 50 0.8074757
45 244.2825 248.16 1000 50 0.57404643
46 248.16 252.0375 1000 50 0.5844841
47 252.0375 255.915 1000 50 0.77901065
48 267.5475 271.425 1000 50 0.9999262
49 283.0575 286.935 1000 50 0.91496134
50 286.935 290.8125 1000 50 0.6016941
51 290.8125 294.69 1000 50 0.5288352
52 294.69 298.5675 1000 50 0.7803428
53 298.5675 302.445 1000 50 0.9457351
54 302.445 306.3225 1000 50 0.8809803
55 314.0775 317.955 1000 50 0.76862615
56 317.955 321.8325 1000 50 0.9179431
57 321.8325 325.71 1000 50 0.64783925
58 325.71 329.5875 1000 50 0.63720423
59 333.465 337.3425 1000 50 0.9080041
60 337.3425 341.21999999999997 1000 50 0.636528
61 341.22 345.0975 1000 50 0.8285956
62 348.975 352.8525 1000 50 0.5541101
63 360.6075 364.485 1000 50 0.9939563
64 364.485 368.3625 1000 50 0.5179203
65 368.3625 372.24 1000 50 0.998485
66 372.24 376.1175 1000 50 0.7255482
67 379.995 383.8725 1000 50 0.6432554
68 387.75 391.6275 1000 50 0.85360736
69 395.505 399.3825 1000 50 0.63949126
70 403.26 407.1375 1000 50 0.9875199
71 407.1375 411.015 1000 50 0.63171786
72 411.015 414.8925 1000 50 0.51271856
73 414.8925 418.77 1000 50 0.8624972
74 418.77 422.6475 1000 50 0.9287566
75 422.6475 426.525 1000 50 0.79819566
76 442.035 445.9125 1000 50 0.9992767
77 445.9125 449.79 1000 50 0.85704327
78 449.79 453.6675 1000 50 0.606248
79 465.3 469.1775 1000 50 0.6945031
80 469.1775 473.055 1000 50 0.91730964
81 476.9325 480.81 1000 50 0.8548426
82 480.81 484.6875 1000 50 0.8609153
83 484.6875 488.565 1000 50 0.53578407
84 492.4425 496.32 1000 50 0.6815063
85 504.075 507.9525 1000 50 0.5493686
86 511.83 515.7075 1000 50 0.57242775
87 515.7075 519.585 1000 50 0.802748
88 519.585 523.4625000000001 1000 50 0.75667137
89 527.34 531.2175000000001 1000 50 0.6283601
90 535.095 538.9725000000001 1000 50 0.58465177
91 538.9725 542.85 1000 50 0.7337384
92 546.7275 550.605 1000 50 0.6839575
93 562.2375 566.115 1000 50 0.6672469
94 593.2575 597.1350000000001 1000 50 0.5644848
95 604.89 608.7675 1000 50 0.78537405
96 624.2775 628.1550000000001 1000 50 0.52095556
97 632.0325 635.9100000000001 1000 50 0.86063546
98 635.91 639.7875 1000 50 0.7444124
99 643.665 647.5425 1000 50 0.59704643
100 647.5425 651.4200000000001 1000 50 0.9363932
101 666.93 670.8075 1000 50 0.78785974
102 678.5625 682.44 1000 50 0.99932384
103 690.195 694.0725000000001 1000 50 0.7276663
104 694.0725 697.95 1000 50 0.6033868
105 697.95 701.8275000000001 1000 50 0.7666113
106 705.705 709.5825000000001 1000 50 0.9996606
107 709.5825 713.46 1000 50 0.87555915
108 717.3375 721.215 1000 50 0.99977213
109 721.215 725.0925000000001 1000 50 0.6706097
110 728.97 732.8475000000001 1000 50 0.99964714
111 736.725 740.6025000000001 1000 50 0.9942172
112 740.6025 744.48 1000 50 0.99729407
113 744.48 748.3575000000001 1000 50 0.98387647
114 748.3575 752.235 1000 50 0.97733885
115 756.1125 759.99 1000 50 0.9908622
116 763.8675 767.745 1000 50 0.9963475
117 775.5 779.3775 1000 50 0.9301784
118 779.3775 783.2550000000001 1000 50 0.8671598
119 783.255 787.1325 1000 50 0.8163382
120 787.1325 791.0100000000001 1000 50 0.67742085
121 794.8875 798.7650000000001 1000 50 0.9999998
122 798.765 802.6425 1000 50 0.9092158
123 802.6425 806.5200000000001 1000 50 0.5064141
124 806.52 810.3975 1000 50 0.9245316
125 810.3975 814.2750000000001 1000 50 0.9987917
126 814.275 818.1525 1000 50 0.98765403
127 818.1525 822.0300000000001 1000 50 0.7059854
128 822.03 825.9075 1000 50 0.9687309
129 825.9075 829.7850000000001 1000 50 0.99980175
130 829.785 833.6625 1000 50 0.9667981
131 833.6625 837.5400000000001 1000 50 0.6459364
132 837.54 841.4175 1000 50 0.99052876
133 845.295 849.1725 1000 50 0.98032534
134 849.1725 853.0500000000001 1000 50 0.9769905
135 864.6825 868.5600000000001 1000 50 0.9989409
136 868.56 872.4375 1000 50 0.65797764
137 876.315 880.1925000000001 1000 50 0.8667758
138 880.1925 884.07 1000 50 0.8123294
139 884.07 887.9475000000001 1000 50 0.9994879
140 887.9475 891.825 1000 50 0.6211881
141 895.7025 899.58 1000 50 0.9882314
142 899.58 903.4575000000001 1000 50 0.988451
143 903.4575 907.335 1000 50 0.8560532
144 907.335 911.2125000000001 1000 50 0.8371353
145 911.2125 915.09 1000 50 0.9963776
146 918.9675 922.845 1000 50 0.99993306
147 930.6 934.4775000000001 1000 50 0.98831177
148 938.355 942.2325000000001 1000 50 0.6520295
149 957.7425 961.62 1000 50 0.6918473
150 961.62 965.4975000000001 1000 50 0.74858236
151 965.4975 969.375 1000 50 0.97066724
152 973.2525 977.1300000000001 1000 50 0.6426704
153 981.0075 984.8850000000001 1000 50 0.8150208
154 984.885 988.7625 1000 50 0.5683353
155 988.7625 992.6400000000001 1000 50 0.581548
156 992.64 996.5175 1000 50 0.77473736
157 1000.395 1004.2725 1000 50 0.950805
158 1004.2725 1008.1500000000001 1000 50 0.53004926
159 1012.0275 1015.9050000000001 1000 50 0.9998928
160 1015.905 1019.7825 1000 50 0.90716743
161 1023.66 1027.5375 1000 50 0.9305084
162 1031.415 1035.2925 1000 50 0.77918565
163 1039.17 1043.0475000000001 1000 50 0.9984765
164 1043.0475 1046.925 1000 50 0.5665361
165 1050.8025 1054.68 1000 50 0.999446
166 1054.68 1058.5575000000001 1000 50 0.5452138
167 1062.435 1066.3125 1000 50 0.99469894
168 1066.3125 1070.19 1000 50 0.5405148
169 1074.0675 1077.9450000000002 1000 50 0.5739461
170 1077.945 1081.8225 1000 50 0.5058043
171 1081.8225 1085.7 1000 50 0.883449
172 1089.5775 1093.4550000000002 1000 50 0.9999634
173 1093.455 1097.3325 1000 50 0.6814424
174 1097.3325 1101.21 1000 50 0.904202
175 1105.0875 1108.9650000000001 1000 50 0.55364376
176 1120.5975 1124.4750000000001 1000 50 0.719839
177 1136.1075 1139.9850000000001 1000 50 0.57873297
178 1147.74 1151.6175 1000 50 0.7904256
179 1151.6175 1155.4950000000001 1000 50 0.88318896
180 1178.76 1182.6375 1000 50 0.88804716
181 1186.515 1190.3925000000002 1000 50 0.6625152
182 1198.1475 1202.025 1000 50 0.9660911
183 1205.9025 1209.78 1000 50 0.91225654
184 1209.78 1213.6575 1000 50 0.7151032
185 1221.4125 1225.29 1000 50 0.9564372
186 1244.6775 1248.555 1000 50 0.722955
187 1248.555 1252.4325000000001 1000 50 0.7033242
188 1252.4325 1256.31 1000 50 0.72211057
189 1256.31 1260.1875 1000 50 0.6124199
190 1260.1875 1264.065 1000 50 0.7847693
191 1264.065 1267.9425 1000 50 0.6674457
192 1267.9425 1271.8200000000002 1000 50 0.76507866
193 1271.82 1275.6975 1000 50 0.8226003
194 1279.575 1283.4525 1000 50 0.5882834
195 1283.4525 1287.3300000000002 1000 50 0.94607145
196 1287.33 1291.2075 1000 50 0.5691646
197 1295.085 1298.9625 1000 50 0.9842685
198 1298.9625 1302.8400000000001 1000 50 0.9851393
199 1310.595 1314.4725 1000 50 0.5915515
200 1314.4725 1318.3500000000001 1000 50 0.56246245
201 1318.35 1322.2275 1000 50 0.996443
202 1326.105 1329.9825 1000 50 0.8811614
203 1329.9825 1333.8600000000001 1000 50 0.9826125
204 1337.7375 1341.615 1000 50 0.9936857
205 1345.4925 1349.3700000000001 1000 50 0.997924
206 1357.125 1361.0025 1000 50 0.88610935
207 1364.88 1368.7575000000002 1000 50 0.5882369
208 1388.145 1392.0225 1000 50 0.6528943
209 1395.9 1399.7775000000001 1000 50 0.7434355
210 1407.5325 1411.41 1000 50 0.53381926
211 1438.5525 1442.43 1000 50 0.7328098
212 1446.3075 1450.185 1000 50 0.65970933
213 1465.695 1469.5725 1000 50 0.539152
214 1485.0825 1488.96 1000 50 0.6864708
215 1492.8375 1496.7150000000001 1000 50 0.6845237