diff --git a/.classpath b/.classpath
index 6086187c..6b05441f 100644
--- a/.classpath
+++ b/.classpath
@@ -6,7 +6,7 @@
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/ojdk-21.0.1">
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/java-21-openjdk-amd64">
<attributes>
<attribute name="module" value="true"/>
<attribute name="maven.pomderived" value="true"/>
diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs
index 7b5e9ffa..3a3537cd 100644
--- a/.settings/org.eclipse.jdt.core.prefs
+++ b/.settings/org.eclipse.jdt.core.prefs
@@ -11,9 +11,9 @@ org.eclipse.jdt.core.codeComplete.staticFinalFieldPrefixes=
org.eclipse.jdt.core.codeComplete.staticFinalFieldSuffixes=
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=17
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=21
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=17
+org.eclipse.jdt.core.compiler.compliance=21
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
@@ -23,4 +23,4 @@ org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning
org.eclipse.jdt.core.compiler.release=enabled
-org.eclipse.jdt.core.compiler.source=17
+org.eclipse.jdt.core.compiler.source=21
diff --git a/build/deb/PAMGuardIcon2.png b/build/deb/PAMGuardIcon2.png
new file mode 100644
index 00000000..ede35fda
Binary files /dev/null and b/build/deb/PAMGuardIcon2.png differ
diff --git a/build/deb/control b/build/deb/control
new file mode 100644
index 00000000..fa4cf8c5
--- /dev/null
+++ b/build/deb/control
@@ -0,0 +1,13 @@
+Package: PAMGuard
+Version: 2.0.1
+Section: java
+Priority: optional
+Architecture: all
+Maintainer: Jamie Macaulay <jdjm@st-andrews.ac.uk>
+Description: Process passive acoustic data for whales, dolphins and other species.
+Icon: pamguard_icon.png
+Depends: openjdk-21-jre
+postinst script:
+ #!/bin/bash
+ # Set JVM options for the application
+ export JAVA_OPTS="-Xmx4g -Xms512m -Dsun.java2d.uiScale=2 -Dawt.useSystemAAFontSettings=on -Dswing.aatext=true"
diff --git a/build/deb/pamguard.desktop b/build/deb/pamguard.desktop
new file mode 100755
index 00000000..74832481
--- /dev/null
+++ b/build/deb/pamguard.desktop
@@ -0,0 +1,8 @@
+[Desktop Entry]
+Name=PAMGuard
+Comment=Application for passive acoustic monitoring v1
+Exec=java -jar /usr/share/pamguard/Pamguard -c
+Icon=/usr/share/pamguard/PAMGuardIcon2.png
+Terminal=true
+Type=Application
+Categories=Application;
diff --git a/build/deb/set-java-propery.sh b/build/deb/set-java-propery.sh
new file mode 100644
index 00000000..1fc57653
--- /dev/null
+++ b/build/deb/set-java-propery.sh
@@ -0,0 +1,8 @@
+[Desktop Entry]
+Name=PAMGuard
+Comment=Application for passive acoustic monitoring
+Exec=java -jar /usr/share/pamguard/Pamguard-2.02.14a.jar -c
+Icon=/path/to/your/icon.png
+Terminal=true
+Type=Application
+Categories=Application;
diff --git a/data.mat b/data.mat
new file mode 100644
index 00000000..08763ca7
Binary files /dev/null and b/data.mat differ
diff --git a/dependency-reduced-pom.xml b/dependency-reduced-pom.xml
index c616628a..41b66cde 100644
--- a/dependency-reduced-pom.xml
+++ b/dependency-reduced-pom.xml
@@ -26,6 +26,71 @@
</resources>
</build>
<profiles>
+ <profile>
+ <id>linux-profile</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.vafer</groupId>
+ <artifactId>jdeb</artifactId>
+ <version>1.11</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>jdeb</goal>
+ </goals>
+ <configuration>
+ <controlDir>${basedir}/build/deb</controlDir>
+ <dataSet>
+ <data>
+ <src>${project.build.directory}/${project.build.finalName}.jar</src>
+ <type>file</type>
+ <mapper>
+ <type>perm</type>
+ <prefix>/usr/share/pamguard</prefix>
+ </mapper>
+ </data>
+ <data>
+ <src>${basedir}/liblinux</src>
+ <type>directory</type>
+ <includes>*.txt</includes>
+ <includes>*.so</includes>
+ <mapper>
+ <type>perm</type>
+ <prefix>/usr/share/pamguard/liblinux</prefix>
+ </mapper>
+ </data>
+ <data>
+ <src>${basedir}/build/deb/PAMGuardIcon2.png</src>
+ <type>file</type>
+ <mapper>
+ <type>perm</type>
+ <prefix>/usr/share/pamguard</prefix>
+ </mapper>
+ </data>
+ <data>
+ <src>${basedir}/build/deb/pamguard.desktop</src>
+ <type>file</type>
+ <mapper>
+ <type>perm</type>
+ <prefix>/usr/share/applications</prefix>
+ </mapper>
+ </data>
+ <data>
+ <type>link</type>
+ <linkName>/usr/share/pamguard/Pamguard</linkName>
+ <linkTarget>/usr/share/pamguard/${project.build.finalName}.jar</linkTarget>
+ <symlink>true</symlink>
+ </data>
+ </dataSet>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
<profile>
<id>macos-profile</id>
<build>
@@ -145,36 +210,11 @@
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.12.1</version>
- <executions>
- <execution>
- <id>default-compile</id>
- <phase>compile</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- <configuration>
- <release>21</release>
- <compilerId>jdt</compilerId>
- </configuration>
- </execution>
- <execution>
- <id>default-testCompile</id>
- <phase>test-compile</phase>
- <goals>
- <goal>testCompile</goal>
- </goals>
- <configuration>
- <release>21</release>
- <compilerId>jdt</compilerId>
- </configuration>
- </execution>
- </executions>
<dependencies>
<dependency>
<groupId>org.eclipse.tycho</groupId>
<artifactId>tycho-compiler-jdt</artifactId>
<version>1.5.1</version>
- <scope>compile</scope>
</dependency>
</dependencies>
<configuration>
@@ -207,17 +247,6 @@
</transformer>
<transformer />
</transformers>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- <exclude>test/resources/**</exclude>
- </excludes>
- </filter>
- </filters>
</configuration>
</execution>
</executions>
@@ -254,7 +283,7 @@
<goal>copy-dependencies</goal>
</goals>
<configuration>
- <outputDirectory>C:\Users\dg50\source\repos\PAMGuardPAMGuard\target/tempDependencies</outputDirectory>
+ <outputDirectory>${project.build.directory}/tempDependencies</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
diff --git a/pom.xml b/pom.xml
index 1e597d64..d6642867 100644
--- a/pom.xml
+++ b/pom.xml
@@ -51,7 +51,106 @@
earlier. Most OS specific profiles are for generating executables and code signing.
Note use help:active-profiles to see which profiles are active -->
<profiles>
-
+
+ <!-- The Linux profile creates a DMG package for installing PAMGuard on
+ Linux Ubuntu based systems-->
+ <!--Note: There is a Maven bug/feature when using profiles when the plugin order is messed up and seemingly quite random.
+ In PAMGuard it essential that the shade plugin is called before the linux plugin. Therefore explicately call
+ the package phase using mvn package shade:shade jdeb:jdeb. Note that although a deb file can be created in Windows it
+ will not have the correct JavaFX libraries and therefore will not work when riun -->
+ <profile>
+ <id>linux-profile</id>
+ <activation>
+ <os>
+ <family>linux</family>
+ </os>
+ </activation>
+ <build>
+ <plugins>
+ <!--Creates a deb file for Linux-->
+ <plugin>
+ <artifactId>jdeb</artifactId>
+ <groupId>org.vafer</groupId>
+ <version>1.11</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>jdeb</goal>
+ </goals>
+ <configuration>
+ <controlDir>${basedir}/build/deb</controlDir>
+ <dataSet>
+ <data>
+ <!--TODO-really the jar file should be named properly but thos would mean we would
+ have to change the name in the desktop file too. For now create a link to the jar file
+ with correct version number-->
+ <src>
+ ${project.build.directory}/${project.build.finalName}.jar</src>
+ <type>file</type>
+ <mapper>
+ <type>perm</type>
+ <prefix>/usr/share/pamguard</prefix>
+ </mapper>
+ </data>
+ <data>
+ <src>${basedir}/liblinux</src>
+ <type>directory</type>
+ <includes>*.txt</includes>
+ <includes>*.so</includes>
+ <mapper>
+ <type>perm</type>
+ <prefix>
+ /usr/share/pamguard/liblinux</prefix>
+ </mapper>
+ </data>
+ <data>
+ <src>
+ ${basedir}/build/deb/PAMGuardIcon2.png</src>
+ <type>file</type>
+ <mapper>
+ <type>perm</type>
+ <prefix>/usr/share/pamguard</prefix>
+ </mapper>
+ </data>
+ <!---THe desktop file will create a PAMGuard icon in the applications tray-->
+ <data>
+ <src>
+ ${basedir}/build/deb/pamguard.desktop</src>
+ <type>file</type>
+ <mapper>
+ <type>perm</type>
+ <prefix>/usr/share/applications</prefix>
+ </mapper>
+ </data>
+ <!---Create a link which is just called Pamguard. This means the .desktop file does not need
+ to be altered depending on the build name-->
+ <data>
+ <type>link</type>
+ <linkName>
+ /usr/share/pamguard/Pamguard</linkName>
+ <linkTarget>
+ /usr/share/pamguard/${project.build.finalName}.jar</linkTarget>
+ <symlink>true</symlink>
+ </data>
+ <!--<data>
+ <src>${project.basedir}/build/deb/set-java-property.sh</src>
+ <type>file</type>
+ <mapper>
+ <type>perm</type>
+ <prefix>/usr/bin</prefix>
+ <filemode>755</filemode>
+ </mapper>
+ </data>-->
+ </dataSet>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
<!-- The MacOS profile creates a DMG package for installing PAMGuard on MacOS-->
<!--Note: There is a Maven bug when using profiles when the plugin order is messed up and seemingly quite random.
In PAMGuard it essential that the shade plugin is called before the macos plugin and so the phase of the
@@ -311,11 +410,10 @@
</execution>
-->
</executions>
- </plugin>
-
+ </plugin>
+
</plugins>
-
<pluginManagement>
<plugins>
<!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
@@ -358,7 +456,52 @@
</profile>
</profiles>
-
+
+ <!-- Plugin which creates a .dmg file for MacOS.
+ <plugin>
+ <groupId>de.perdian.maven.plugins</groupId>
+ <artifactId>macosappbundler-maven-plugin</artifactId>
+ <version>1.21.1</version>
+ <configuration>
+ <plist>
+ <JVMMainClassName>pamguard.Pamguard</JVMMainClassName>
+ <CFBundleIconFile>src/Resources/PamguardIcon2.icns</CFBundleIconFile>
+ <CFBundleDisplayName>PAMGuard</CFBundleDisplayName>
+ <CFBundleDevelopmentRegion>English</CFBundleDevelopmentRegion>
+ <CFBundleURLTypes>
+ <string>msa</string>
+ </CFBundleURLTypes>
+ <JVMVersion>21+</JVMVersion>
+ <JVMArguments>
+ <string>-c</string>
+ </JVMArguments>
+ </plist>
+ <dmg>
+ <generate>true</generate>
+ <additionalResources>
+ <additionalResource>
+ <directory>src/target/bundle/</directory>
+ </additionalResource>
+ </additionalResources>
+ </dmg>
+ <jdk>
+ <include>false</include>
+ <location>/Library/Java/JavaVirtualMachines/amazon-corretto-21.jdk</location>
+ </jdk>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>bundle</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>-->
+
+
+
+
<reporting>
@@ -442,7 +585,7 @@
<groupId>io.github.macster110</groupId>
<artifactId>jpamutils</artifactId>
<version>0.0.59f</version>
- <!-- com.github.psambit9791:wavfile:jar:0.1 pulls in various junit dependencies which breaks our own testing -->
+ <!-- com.github.psambit9791:wavfile:jar:0.1 pulls in various junit dependencies which breaks our own testing -->
<!--<exclusions>
<exclusion>
<groupId>org.junit.platform</groupId>
@@ -610,7 +753,7 @@
</exclusions>
</dependency>
-<!-- html to markdown conversion https://github.com/furstenheim/copy-down -->
+ <!-- html to markdown conversion https://github.com/furstenheim/copy-down -->
<dependency>
<groupId>io.github.furstenheim</groupId>
<artifactId>copy_down</artifactId>
@@ -997,7 +1140,7 @@
<version>2.1.1</version>
</dependency>
-<!--
+ <!--
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
@@ -1065,7 +1208,7 @@
-->
<dependency>
<groupId>org.pamguard</groupId>
- <artifactId>x3</artifactId>
+ <artifactId>X3</artifactId>
<version>2.2.8</version>
</dependency>
@@ -1154,7 +1297,6 @@
</dependency>
-
<!-- Test dependencies -->
<dependency>
<groupId>org.junit.jupiter</groupId>
diff --git a/src/Localiser/detectionGroupLocaliser/GroupLocResult.java b/src/Localiser/detectionGroupLocaliser/GroupLocResult.java
index 4b361768..e45fe2bd 100644
--- a/src/Localiser/detectionGroupLocaliser/GroupLocResult.java
+++ b/src/Localiser/detectionGroupLocaliser/GroupLocResult.java
@@ -106,10 +106,6 @@ public class GroupLocResult implements Comparable<GroupLocResult>, LocalisationC
this.chi2 = chi2;
}
- public GroupLocResult(double[] result, double[] resultErrors, int side, double chi2) {
- // TODO Auto-generated constructor stub
- }
-
/**
* @return the latLong
*/
diff --git a/src/PamUtils/PamArrayUtils.java b/src/PamUtils/PamArrayUtils.java
index 89bc52a2..cb371645 100644
--- a/src/PamUtils/PamArrayUtils.java
+++ b/src/PamUtils/PamArrayUtils.java
@@ -1381,7 +1381,25 @@ public class PamArrayUtils {
}
-
+ /**
+ * Convert a matrix to a
+ * @param matrix - the MAT file matrix
+ * @return double[][] array of results
+ */
+ public static float[][] matrix2arrayF(Matrix matrix) {
+ if (matrix==null) return null;
+
+ float[][] arrayOut = new float[matrix.getNumRows()][];
+ float[] arrayRow;
+ for (int i=0; i<matrix.getNumRows(); i++) {
+ arrayRow=new float[matrix.getNumCols()];
+ for (int j=0; j<matrix.getNumCols(); j++) {
+ arrayRow[j] = matrix.getFloat(i, j);
+ }
+ arrayOut[i]=arrayRow;
+ }
+ return arrayOut;
+ }
diff --git a/src/dataModelFX/DataModelConnectPane.java b/src/dataModelFX/DataModelConnectPane.java
index bf3f02cf..9b4d83ea 100644
--- a/src/dataModelFX/DataModelConnectPane.java
+++ b/src/dataModelFX/DataModelConnectPane.java
@@ -54,13 +54,18 @@ public class DataModelConnectPane extends ConnectionPane {
@Override
public void handle(DragEvent event)
{
+
+// System.out.println("START drag over node drag dropped" + dataModelPaneFX.getDraggingStructure());
+
final Dragboard dragboard = event.getDragboard();
if (dragboard.hasString()
&& dataModelPaneFX.getModuleDragKey().equals(dragboard.getString())
&& dataModelPaneFX.getDraggingModule().get() != null || dataModelPaneFX.getDraggingStructure()!=null)
{
- event.acceptTransferModes(TransferMode.MOVE);
- event.consume();
+// System.out.println("ACCEPT drag over node drag dropped" + dataModelPaneFX.getDraggingStructure());
+
+ event.acceptTransferModes(TransferMode.ANY);
+ //event.consume(); //causesd issues with dropping nodes not being detected
}
}
});
@@ -70,6 +75,9 @@ public class DataModelConnectPane extends ConnectionPane {
@Override
public void handle(DragEvent event)
{
+
+ System.out.println("Add Some Node drag dropped: " + dataModelPaneFX.getDraggingStructure());
+
final Dragboard dragboard = event.getDragboard();
if (dragboard.hasString()
&& dataModelPaneFX.getModuleDragKey().equals(dragboard.getString())
@@ -80,6 +88,7 @@ public class DataModelConnectPane extends ConnectionPane {
ModuleRectangle moduleRect = dataModelPaneFX.getDraggingModule().get();
+// System.out.println("Add Connection Node drag dropped");
dataModelPaneFX.getDraggingModule().set(null);
dataModelPaneFX.getConnectionNodeFactory().addNewModule(moduleRect.getPamModuleInfo(), event.getX(), event.getY());
@@ -88,6 +97,7 @@ public class DataModelConnectPane extends ConnectionPane {
if (dragboard.hasString()
&& dataModelPaneFX.getModuleDragKey().equals(dragboard.getString())
&& dataModelPaneFX.getDraggingStructure().get() != null){
+// System.out.println("Add Structure Node drag dropped");
dataModelPaneFX.getConnectionNodeFactory().addNewStructure(dataModelPaneFX.getDraggingStructure().get(), event.getX(), event.getY());
dataModelPaneFX.getDraggingStructure().set(null);
diff --git a/src/dataModelFX/connectionNodes/ConnectionNodeFactory.java b/src/dataModelFX/connectionNodes/ConnectionNodeFactory.java
index 23e14bc0..9e1d7cba 100644
--- a/src/dataModelFX/connectionNodes/ConnectionNodeFactory.java
+++ b/src/dataModelFX/connectionNodes/ConnectionNodeFactory.java
@@ -121,8 +121,7 @@ public class ConnectionNodeFactory {
* @param y - the pixel y co-ordinate
*/
public StandardConnectionNode addNewStructure(StructureRectangle structureRectangle, double x, double y) {
- // System.out.println("DataModelConnectPane: add new structure " + structureRectangle.getStructureType()
- // + " no. nodes: " + this.getConnectionNodes().size());
+ System.out.println("DataModelConnectPane: add new structure " + structureRectangle.getStructureType() );
StandardConnectionNode connectionNode = createConnectionNode(structureRectangle.getStructureType(), null);
if (connectionNode!=null) {
diff --git a/src/dataPlotsFX/data/generic/GenericDataPlotInfo.java b/src/dataPlotsFX/data/generic/GenericDataPlotInfo.java
index 7d00c0dc..cf2ab947 100644
--- a/src/dataPlotsFX/data/generic/GenericDataPlotInfo.java
+++ b/src/dataPlotsFX/data/generic/GenericDataPlotInfo.java
@@ -136,6 +136,8 @@ public class GenericDataPlotInfo extends TDDataInfoFX {
System.out.println("GenericDataPlotInfo: Single frequency measure in data unit " + pamDataUnit.toString());
}
+ //System.out.println("Frequency: " + f[0] + " " + f[1] + " " + pamDataUnit);
+
// draw a frequency box.
double y0 = tdProjector.getYPix(f[0]);
double y1 = tdProjector.getYPix(f[1]);
diff --git a/src/export/MLExport/MLDetectionsManager.java b/src/export/MLExport/MLDetectionsManager.java
index 052539bd..2fbb5db4 100644
--- a/src/export/MLExport/MLDetectionsManager.java
+++ b/src/export/MLExport/MLDetectionsManager.java
@@ -153,6 +153,9 @@ public class MLDetectionsManager implements PamDataUnitExporter {
public boolean hasCompatibleUnits(List<PamDataUnit> dataUnits) {
//first need to figure out how many data units there are.
for (int j=0; j<dataUnits.size(); j++){
+
+
+
if (hasCompatibleUnits(dataUnits.get(j).getClass())) return true;
}
return false;
diff --git a/src/group3dlocaliser/Group3DProcess.java b/src/group3dlocaliser/Group3DProcess.java
index 51a80832..0a23ad05 100644
--- a/src/group3dlocaliser/Group3DProcess.java
+++ b/src/group3dlocaliser/Group3DProcess.java
@@ -232,7 +232,7 @@ public class Group3DProcess extends PamProcess implements DetectionGroupMonitor
}
}
-// System.out.println("Ran localisation " + i + " " + localiserAlgorithm3D.getName() + " got: " + abstractLocalisation.getLatLong(0) + " " + abstractLocalisation.getHeight(0) + " Error: " + abstractLocalisation.getLocError(0));
+ System.out.println("Ran localisation " + i + " " + localiserAlgorithm3D.getName() + " got: " + abstractLocalisation.getLatLong(0) + " " + abstractLocalisation.getHeight(0) + " Error: " + abstractLocalisation.getLocError(0));
if (abstractLocalisation instanceof GroupLocalisation) {
groupLocalisation = (GroupLocalisation) abstractLocalisation;
diff --git a/src/group3dlocaliser/algorithm/hyperbolic/HyperbolicLocaliser.java b/src/group3dlocaliser/algorithm/hyperbolic/HyperbolicLocaliser.java
index f9483bad..d0717d57 100644
--- a/src/group3dlocaliser/algorithm/hyperbolic/HyperbolicLocaliser.java
+++ b/src/group3dlocaliser/algorithm/hyperbolic/HyperbolicLocaliser.java
@@ -388,7 +388,8 @@ public class HyperbolicLocaliser extends TOADBaseAlgorithm {
PamVector centre = geometry.getGeometricCentre();
int[] hydrophones = toadInformation.getHydrophoneList();
double xArray[] = new double[geom.length];
- for (int i = 0; i < geom.length; i++) {
+ //System.out.println("Hyperbolic loc: Geom length: " + hydrophones.length + " " + geom.length);
+ for (int i = 0; i < hydrophones.length; i++) {
xArray[i] = geom[hydrophones[i]].sub(centre).dotProd(arrayAxes);
}
@@ -581,6 +582,8 @@ public class HyperbolicLocaliser extends TOADBaseAlgorithm {
//// glr.setModel(model);
// return groupLocalisation;
LinearLocalisation linLoc = new LinearLocalisation(groupDataUnit, toadInformation.getHydrophoneMap(), geometry.getArrayAxes(), angle, r);
+ linLoc.setChi2(chiData);
+
GpsData refPos = new GpsData(geometry.getReferenceGPS().addDistanceMeters(geometry.getGeometricCentre()));
linLoc.setReferencePosition(refPos);
return linLoc;
diff --git a/src/group3dlocaliser/algorithm/toadbase/TOADBaseAlgorithm.java b/src/group3dlocaliser/algorithm/toadbase/TOADBaseAlgorithm.java
index bb09e5c5..c05cf5b4 100644
--- a/src/group3dlocaliser/algorithm/toadbase/TOADBaseAlgorithm.java
+++ b/src/group3dlocaliser/algorithm/toadbase/TOADBaseAlgorithm.java
@@ -132,11 +132,14 @@ abstract public class TOADBaseAlgorithm extends LocaliserAlgorithm3D {
// if (superDetection.getSubDetection(0).getUID() == 9035004477L) {
// System.out.println("Found it");
// }
+
+
TOADInformation toadInformation = toadCalculator.getTOADInformation(superDetection.getSubDetections(), sampleRate, allChannels, geometry);
+
boolean toadOK = checkTOADInformation(toadInformation);
-// System.out.println("ToadOK: " + toadOK + " toadInformation: " + toadInformation + " allChannels: " + allChannels + " geometry: " + geometry);
+ //System.out.println("ToadOK: " + toadOK + " toadInformation: " + toadInformation + " allChannels: " + allChannels + " geometry: " + geometry.getGeometry() + "n sub: " + superDetection.getSubDetectionsCount() );
if (!toadOK) {
return null;
diff --git a/src/group3dlocaliser/algorithm/toadmimplex/LinearGroupLocResult.java b/src/group3dlocaliser/algorithm/toadmimplex/LinearGroupLocResult.java
new file mode 100644
index 00000000..c31a418c
--- /dev/null
+++ b/src/group3dlocaliser/algorithm/toadmimplex/LinearGroupLocResult.java
@@ -0,0 +1,17 @@
+package group3dlocaliser.algorithm.toadmimplex;
+
+import Localiser.detectionGroupLocaliser.GroupLocResult;
+import group3dlocaliser.localisation.LinearLocalisation;
+
+
+/**
+ * Wrapper to make a linear localisation into a group localisation
+ */
+public class LinearGroupLocResult extends GroupLocResult {
+
+ public LinearGroupLocResult(LinearLocalisation linearLocalisation) {
+ super(linearLocalisation.getReferencePosition(), 0, linearLocalisation.getChi2());
+
+ }
+
+}
diff --git a/src/group3dlocaliser/algorithm/toadmimplex/ToadMimplexLocaliser.java b/src/group3dlocaliser/algorithm/toadmimplex/ToadMimplexLocaliser.java
index 797fd542..6defafc5 100644
--- a/src/group3dlocaliser/algorithm/toadmimplex/ToadMimplexLocaliser.java
+++ b/src/group3dlocaliser/algorithm/toadmimplex/ToadMimplexLocaliser.java
@@ -17,6 +17,7 @@ import group3dlocaliser.algorithm.toadmcmc.MCMCLocaliserPane;
import group3dlocaliser.algorithm.toadmcmc.ToadMCMCLocaliser;
import group3dlocaliser.algorithm.toadsimplex.ToadSimplexLocaliser;
import group3dlocaliser.grouper.DetectionGroupedSet;
+import group3dlocaliser.localisation.LinearLocalisation;
/**
@@ -94,7 +95,7 @@ public class ToadMimplexLocaliser extends ToadMCMCLocaliser {
*/
public DetectionGroupedSet preFilterLoc(DetectionGroupedSet preGroups) {
- System.out.println("Pre filter groups: " + preGroups.getNumGroups());
+ //System.out.println("Pre filter groups: " + preGroups.getNumGroups());
MimplexParams params = (MimplexParams) group3dLocaliser.getLocaliserAlgorithmParams(this).getAlgorithmParameters();
@@ -105,12 +106,12 @@ public class ToadMimplexLocaliser extends ToadMCMCLocaliser {
return preGroups;
}
- //no need to do a y more processing.
+ //no need to do a any more processing.
if (preGroups.getNumGroups()<=2 && params.useFirstCombination) {
return preGroups;
}
- //localiser using both hyperbolic and the
+ //localiser using both hyperbolic and simplex the
// will have to make a data unit for each group now...
Group3DDataUnit[] group3dDataUnits = new Group3DDataUnit[preGroups.getNumGroups()];
@@ -121,18 +122,22 @@ public class ToadMimplexLocaliser extends ToadMCMCLocaliser {
group3dDataUnits[i] = new Group3DDataUnit(preGroups.getGroup(i));
+ //System.out.println("ToadMImplex. group3dDataUnits[i]: " + group3dDataUnits[i] );
- GroupLocalisation preAbstractLocalisation = null;
+ AbstractLocalisation preAbstractLocalisation = null;
double minChi2 = Double.MAX_VALUE;
GroupLocResult locResult = null;
for (LocaliserModel model: preLocaliserModels) {
+
+ //System.out.println("ToadMImplex. model: " + model );
+
locResult = null;
minChi2=Double.MAX_VALUE;
preAbstractLocalisation = null; //must reset this.
try {
- preAbstractLocalisation = (GroupLocalisation) model.runModel(group3dDataUnits[i], null, false);
+ preAbstractLocalisation = model.runModel(group3dDataUnits[i], null, false);
}
catch (Exception e) {
System.out.println("Mimplex pre filter loclaisation failed");
@@ -142,12 +147,23 @@ public class ToadMimplexLocaliser extends ToadMCMCLocaliser {
// System.out.println("Pre-localisation result: " + locResult + " " + model.getName() + "N units: " + preGroups.getGroup(i).size());
if (preAbstractLocalisation!=null) {
+
+ if (preAbstractLocalisation instanceof GroupLocalisation) {
//now iterate through the potential ambiguities (this is a bit redunadant with Simplex and Hyperbolic)
- for (GroupLocResult groupLocResult: preAbstractLocalisation.getGroupLocResults()) {
+ for (GroupLocResult groupLocResult: ((GroupLocalisation) preAbstractLocalisation).getGroupLocResults()) {
if (groupLocResult.getChi2()<minChi2) {
locResult = groupLocResult;
}
}
+ }
+ if (preAbstractLocalisation instanceof LinearLocalisation) {
+ //if a linear vertical array (exactly) then will return a linear localisation. Need to make this into
+ //a group localisation to satisfy the mimplex localiser.
+ if (((LinearLocalisation) preAbstractLocalisation).getChi2()<minChi2) {
+ locResult = new LinearGroupLocResult(((LinearLocalisation) preAbstractLocalisation));
+ }
+ }
+
}
}
diff --git a/src/group3dlocaliser/grouper/DetectionGrouper.java b/src/group3dlocaliser/grouper/DetectionGrouper.java
index 5e305ada..58779bac 100644
--- a/src/group3dlocaliser/grouper/DetectionGrouper.java
+++ b/src/group3dlocaliser/grouper/DetectionGrouper.java
@@ -244,6 +244,7 @@ public class DetectionGrouper {
*/
public synchronized void closeMotherGroup() {
processFirstGroup(motherGroup);
+ if (maxInterGroupSamples==null) return;
motherGroup = new FirstGrouping(maxInterGroupSamples.length, 0, null);
}
diff --git a/src/group3dlocaliser/localisation/LinearLocalisation.java b/src/group3dlocaliser/localisation/LinearLocalisation.java
index fc923f3c..1e272363 100644
--- a/src/group3dlocaliser/localisation/LinearLocalisation.java
+++ b/src/group3dlocaliser/localisation/LinearLocalisation.java
@@ -5,6 +5,7 @@ import Localiser.detectionGroupLocaliser.LocalisationChi2;
import PamDetection.AbstractLocalisation;
import PamDetection.LocContents;
import PamguardMVC.PamDataUnit;
+import group3dlocaliser.algorithm.Chi2Data;
import pamMaths.PamVector;
public class LinearLocalisation extends AbstractLocalisation implements LocalisationChi2{
@@ -12,6 +13,7 @@ public class LinearLocalisation extends AbstractLocalisation implements Localisa
private double[] angles;
private Double range;
private GpsData referencePosition;
+ private Chi2Data chi2Dat;
public LinearLocalisation(PamDataUnit pamDataUnit, int referenceHydrophones, PamVector[] arrayAxes, double bearing, Double range) {
super(pamDataUnit, LocContents.HAS_BEARING | LocContents.HAS_AMBIGUITY, referenceHydrophones);
@@ -42,8 +44,7 @@ public class LinearLocalisation extends AbstractLocalisation implements Localisa
@Override
public Double getChi2() {
- // TODO Auto-generated method stub
- return null;
+ return chi2Dat.getChi2();
}
@Override
@@ -84,6 +85,14 @@ public class LinearLocalisation extends AbstractLocalisation implements Localisa
public void setReferencePosition(GpsData referencePosition) {
this.referencePosition = referencePosition;
}
+
+ /**
+ * Set Chi2 data
+ * @param chi2dat
+ */
+ public void setChi2(Chi2Data chi2dat) {
+ this.chi2Dat = chi2dat;
+ }
diff --git a/src/group3dlocaliser/offline/Group3DOfflineTask.java b/src/group3dlocaliser/offline/Group3DOfflineTask.java
index ab525a7b..b7f161bf 100644
--- a/src/group3dlocaliser/offline/Group3DOfflineTask.java
+++ b/src/group3dlocaliser/offline/Group3DOfflineTask.java
@@ -58,7 +58,13 @@ public class Group3DOfflineTask extends OfflineTask<PamDataUnit>{
@Override
public boolean processDataUnit(PamDataUnit dataUnit) {
+ try {
group3DProcess.newData(null, dataUnit);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ }
+ //System.out.println("New data unit added: " +dataUnit);
return false;
}
diff --git a/src/rawDeepLearningClassifier/dlClassification/DLClassifyProcess.java b/src/rawDeepLearningClassifier/dlClassification/DLClassifyProcess.java
index a65666a1..c6d1f4e9 100644
--- a/src/rawDeepLearningClassifier/dlClassification/DLClassifyProcess.java
+++ b/src/rawDeepLearningClassifier/dlClassification/DLClassifyProcess.java
@@ -210,7 +210,7 @@ public class DLClassifyProcess extends PamProcess {
*/
@Override
public void newData(PamObservable obs, PamDataUnit pamRawData) {
- //System.out.println("NEW SEGMENTER DATA: " + PamCalendar.formatDateTime2(pamRawData.getTimeMilliseconds(), "dd MMM yyyy HH:mm:ss.SSS", false) + " " + pamRawData.getUID() + " " + pamRawData.getChannelBitmap() + " " + pamRawData);
+ //System.out.println("NEW SEGMENTER DATA: " + PamCalendar.formatDateTime2(pamRawData.getTimeMilliseconds(), "dd MMM yyyy HH:mm:ss.SSS", false) + " " + pamRawData.getUID() + " " + pamRawData.getChannelBitmap() + " " + pamRawData);
//if grouped data then just run the classifier on the group - do not try and create a buffer.
if (pamRawData instanceof SegmenterDetectionGroup) {
@@ -618,10 +618,11 @@ public class DLClassifyProcess extends PamProcess {
*/
public void forceRunClassifier(PamDataUnit dataUnit) {
-// System.out.println("CLASSIFICATION BUFFER: " + classificationBuffer.size());
+ //System.out.println("CLASSIFICATION BUFFER: " + classificationBuffer.size());
if (this.classificationBuffer.size()>0) {
if (classificationBuffer.get(0) instanceof GroupedRawData) {
+ //System.out.println("Run raw model on: " + classificationBuffer.get(0) + " " + classificationBuffer.get(1));
runRawModel(); //raw data or raw data units
}
if (classificationBuffer.get(0) instanceof SegmenterDetectionGroup) {
@@ -689,10 +690,12 @@ public class DLClassifyProcess extends PamProcess {
DataUnitBaseData basicData = groupDataBuffer.get(0).getBasicData().clone();
basicData.setMillisecondDuration(1000.*rawdata[0].length/this.sampleRate);
basicData.setSampleDuration((long) (groupDataBuffer.size()*dlControl.getDLParams().rawSampleSize));
+
// System.out.println("Model result: " + modelResult.size());
DLDetection dlDetection = new DLDetection(basicData, rawdata, getSampleRate());
addDLAnnotation(dlDetection,modelResult);
+ dlDetection.setFrequency(new double[] {0, this.getSampleRate()/2});
//create the data unit
return dlDetection;
diff --git a/src/rawDeepLearningClassifier/dlClassification/archiveModel/ArchiveModelWorker.java b/src/rawDeepLearningClassifier/dlClassification/archiveModel/ArchiveModelWorker.java
index bf24655f..81619f8c 100644
--- a/src/rawDeepLearningClassifier/dlClassification/archiveModel/ArchiveModelWorker.java
+++ b/src/rawDeepLearningClassifier/dlClassification/archiveModel/ArchiveModelWorker.java
@@ -139,9 +139,8 @@ public class ArchiveModelWorker extends GenericModelWorker {
System.out.println(modelParams.dlTransforms);
ArrayList<DLTransform> transforms = DLTransformsFactory.makeDLTransforms(modelParams.dlTransforms);
-// ///HACK here for now to fix an issue with dB and Ketos transforms having zero length somehow...
// for (int i=0; i<modelParams.dlTransforms.size(); i++) {
-// System.out.println(modelParams.dlTransforms.get(i));
+// System.out.println(modelParams.dlTransforms.get(i).toString());
// }
//only load new transforms if defaults are selected
diff --git a/src/rawDeepLearningClassifier/dlClassification/genericModel/DLModelWorker.java b/src/rawDeepLearningClassifier/dlClassification/genericModel/DLModelWorker.java
index 04eaa946..a956d0d4 100644
--- a/src/rawDeepLearningClassifier/dlClassification/genericModel/DLModelWorker.java
+++ b/src/rawDeepLearningClassifier/dlClassification/genericModel/DLModelWorker.java
@@ -11,6 +11,7 @@ import org.jamdev.jdl4pam.transforms.DLTransformsFactory;
import org.jamdev.jdl4pam.utils.DLUtils;
import org.jamdev.jpamutils.wavFiles.AudioData;
+import PamUtils.PamArrayUtils;
import PamguardMVC.PamDataUnit;
import rawDeepLearningClassifier.DLControl;
import rawDeepLearningClassifier.DLStatus;
@@ -82,10 +83,17 @@ public abstract class DLModelWorker<T> {
DLTransform transform = modelTransforms.get(0);
for (int i =0; i<modelTransforms.size(); i++) {
transform = modelTransforms.get(i).transformData(transform);
+
// //TEMP
// if (transform instanceof FreqTransform) {
// transformedData = ((FreqTransform) transform).getSpecTransfrom().getTransformedData();
// System.out.println("DLModelWorker: transform : " + modelTransforms.get(i).getDLTransformType() + " "+ i + transformedData.length + " " + transformedData[0].length + " minmax: " + PamArrayUtils.minmax(transformedData)[0] + " " + PamArrayUtils.minmax(transformedData)[1]);
+// }
+
+// //TEMP
+// if (transform instanceof WaveTransform) {
+// transformedData1 = ((WaveTransform) transform).getWaveData().getScaledSampleAmplitudes();
+// System.out.println("DLModelWorker: transform : " + modelTransforms.get(i).getDLTransformType() + " "+ i + " " + transformedData1.length + " " + PamArrayUtils.minmax(transformedData1)[0] + " " + PamArrayUtils.minmax(transformedData1)[1]);
// }
}
diff --git a/src/rawDeepLearningClassifier/dlClassification/genericModel/GenericModelWorker.java b/src/rawDeepLearningClassifier/dlClassification/genericModel/GenericModelWorker.java
index 3664fc1b..3fa456a7 100644
--- a/src/rawDeepLearningClassifier/dlClassification/genericModel/GenericModelWorker.java
+++ b/src/rawDeepLearningClassifier/dlClassification/genericModel/GenericModelWorker.java
@@ -104,6 +104,7 @@ public class GenericModelWorker extends DLModelWorker<StandardPrediction> {
}
setModelTransforms(genericParams.dlTransfroms);
+
//is this a waveform or a spectrogram model?
setWaveFreqModel(genericParams);
diff --git a/src/rawDeepLearningClassifier/layoutFX/DLSettingsPane.java b/src/rawDeepLearningClassifier/layoutFX/DLSettingsPane.java
index 4bcc7618..57d03d98 100644
--- a/src/rawDeepLearningClassifier/layoutFX/DLSettingsPane.java
+++ b/src/rawDeepLearningClassifier/layoutFX/DLSettingsPane.java
@@ -583,10 +583,11 @@ public class DLSettingsPane extends SettingsPane<RawDLParams>{
@Override
public void setParams(RawDLParams currParams) {
-
- sourcePane.setParams(currParams.groupedSourceParams);
+
+
sourcePane.sourceChanged();
sourcePane.setSourceList();
+ sourcePane.setParams(currParams.groupedSourceParams);
dlControl.createDataSelector(sourcePane.getSource());
diff --git a/src/rawDeepLearningClassifier/layoutFX/dlTransfroms/PeakTrimTransformPane.java b/src/rawDeepLearningClassifier/layoutFX/dlTransfroms/PeakTrimTransformPane.java
index 5a7bdcef..e7cf805c 100644
--- a/src/rawDeepLearningClassifier/layoutFX/dlTransfroms/PeakTrimTransformPane.java
+++ b/src/rawDeepLearningClassifier/layoutFX/dlTransfroms/PeakTrimTransformPane.java
@@ -59,7 +59,7 @@ public class PeakTrimTransformPane extends DLTransformPane {
});
//spinner for changing filter order.
- targetLenSpinner = new Spinner<Integer>(1,50,4,1);
+ targetLenSpinner = new Spinner<Integer>(1,Integer.MAX_VALUE,4,1);
targetLenSpinner.valueProperty().addListener((obsVal, oldVal, newVal)->{
this.notifySettingsListeners();
});
diff --git a/src/rawDeepLearningClassifier/offline/DLOfflineTask.java b/src/rawDeepLearningClassifier/offline/DLOfflineTask.java
index 4d96db4e..61265df6 100644
--- a/src/rawDeepLearningClassifier/offline/DLOfflineTask.java
+++ b/src/rawDeepLearningClassifier/offline/DLOfflineTask.java
@@ -1,5 +1,7 @@
package rawDeepLearningClassifier.offline;
+import java.util.ListIterator;
+
import PamController.PamController;
import PamguardMVC.PamDataUnit;
import PamguardMVC.PamObservable;
@@ -8,6 +10,7 @@ import dataMap.OfflineDataMapPoint;
import matchedTemplateClassifer.MTClassifierControl;
import offlineProcessing.OfflineTask;
import rawDeepLearningClassifier.DLControl;
+import rawDeepLearningClassifier.segmenter.GroupedRawData;
import rawDeepLearningClassifier.segmenter.SegmenterDetectionGroup;
import rawDeepLearningClassifier.segmenter.SegmenterProcess;
@@ -48,7 +51,7 @@ public class DLOfflineTask extends OfflineTask<PamDataUnit<?,?>>{
@Override
public boolean processDataUnit(PamDataUnit<?, ?> dataUnit) {
// System.out.println("--------------");
- // System.out.println("Offline task start: " + dataUnit.getUpdateCount() + " UID " + dataUnit.getUID());
+ //System.out.println("Offline task start: " + dataUnit.getUpdateCount() + " UID " + dataUnit.getUID() + " " + dlControl.getDLParams().enableSegmentation);
boolean saveBinary = false;
try {
@@ -79,9 +82,16 @@ public class DLOfflineTask extends OfflineTask<PamDataUnit<?,?>>{
//detection has been added we force the classifier to run on all the segments generated from
//the raw data.
- //Process a data unit
+ //Process a data unit within the segmenter
dlControl.getSegmenter().newData(dataUnit);
-
+
+ //System.out.println("Segments: " + dlControl.getSegmenter().getSegmenterDataBlock().getUnitsCount());
+ //need to add the segmenter data units into the classification buffer
+ ListIterator<GroupedRawData> iterator = dlControl.getSegmenter().getSegmenterDataBlock().getListIterator(0);
+ while (iterator.hasNext()) {
+ dlControl.getDLClassifyProcess().newData(dlControl.getSegmenter().getSegmenteGroupDataBlock(), iterator.next());
+ }
+
//force click data save
dlControl.getDLClassifyProcess().forceRunClassifier(dataUnit);
diff --git a/src/rawDeepLearningClassifier/segmenter/SegmenterProcess.java b/src/rawDeepLearningClassifier/segmenter/SegmenterProcess.java
index 9f481cc1..87899d6b 100644
--- a/src/rawDeepLearningClassifier/segmenter/SegmenterProcess.java
+++ b/src/rawDeepLearningClassifier/segmenter/SegmenterProcess.java
@@ -96,7 +96,7 @@ public class SegmenterProcess extends PamProcess {
segmenterDataBlock = new SegmenterDataBlock("Segmented Raw Data", this,
dlControl.getDLParams().groupedSourceParams.getChanOrSeqBitmap());
- segmenterGroupDataBlock = new SegmenterGroupDataBlock("Segmented data units", this,
+ segmenterGroupDataBlock = new SegmenterGroupDataBlock("Segmented Group Data", this,
dlControl.getDLParams().groupedSourceParams.getChanOrSeqBitmap());
addOutputDataBlock(segmenterDataBlock);
@@ -241,7 +241,7 @@ public class SegmenterProcess extends PamProcess {
newRawDataUnit(pamRawData);
}
else if (pamRawData instanceof ClickDetection) {
- newClickData( pamRawData);
+ newClickData(pamRawData);
}
else if (pamRawData instanceof ClipDataUnit) {
newClipData(pamRawData);
@@ -491,7 +491,7 @@ public class SegmenterProcess extends PamProcess {
public void newClickData(PamDataUnit pamRawData) {
//the raw data units should appear in sequential channel order
- // System.out.println("New raw data in: chan: " + PamUtils.getSingleChannel(pamRawData.getChannelBitmap()) + " Size: " + pamRawData.getSampleDuration());
+ //System.out.println("New raw data in: chan: " + PamUtils.getSingleChannel(pamRawData.getChannelBitmap()) + " Size: " + pamRawData.getSampleDuration());
ClickDetection clickDataUnit = (ClickDetection) pamRawData;
@@ -542,6 +542,12 @@ public class SegmenterProcess extends PamProcess {
//segment the data unit into different chunks.
newRawData(pamDataUnit,
rawDataChunk[i], chans[i], dlControl.getDLParams().rawSampleSize, dlControl.getDLParams().sampleHop, true);
+ //the way that the newRawdata works is it waits for the next chunk and copies all relevant bits
+ //from previous chunks into segments. This is fine for continuous data but means that chunks of data
+ //don't get their last hop...
+
+ //got to save the last chunk of raw data -even if the segment has not been filled.
+ saveRawGroupData(true);
}
else {
// //send the whole data chunk to the deep learning unit
@@ -551,13 +557,8 @@ public class SegmenterProcess extends PamProcess {
// pamDataUnit.getStartSample(), rawDataChunk[i].length, rawDataChunk[i].length);
}
- //the way that the newRawdata works is it waits for the next chunk and copies all relevant bits
- //from previous chunks into segments. This is fine for continuous data but means that chunks of data
- //don't get their last hop...
}
- //got to save the last chunk of raw data -even if the segment has not been filled.
- saveRawGroupData(true);
}
@@ -600,7 +601,7 @@ public class SegmenterProcess extends PamProcess {
long timeMilliseconds = unit.getTimeMilliseconds();
long startSampleTime = unit.getStartSample();
- //System.out.println("Segmenter: RawDataIn: chan: 1 " + getSourceParams().countChannelGroups() + currentRawChunks);
+ //System.out.println("Segmenter: RawDataIn: chan: 1 " + getSourceParams().countChannelGroups() + currentRawChunks + " rawSampleSize " + rawSampleSize + " rawSampleHop: " +rawSampleHop);
if (currentRawChunks==null) {
System.err.println("Current raw chunk arrays are null");
@@ -753,6 +754,7 @@ public class SegmenterProcess extends PamProcess {
* @param forceSave - true to also save the remaining unfilled segment.
*/
private void saveRawGroupData(boolean forceSave) {
+ //System.out.println("Segmenter process: saveRawGroupData(boolean forceSave)");
for (int i=0; i<getSourceParams().countChannelGroups(); i++) {
saveRawGroupData(i, forceSave);
}
@@ -771,6 +773,7 @@ public class SegmenterProcess extends PamProcess {
* @param i - the group index.
*/
private void saveRawGroupData(int i) {
+ //System.out.println("Segmenter process: saveRawGroupData(int i)");
saveRawGroupData(i, false);
}
diff --git a/src/test/rawDeepLearningClassifier/ClickDLTest.java b/src/test/rawDeepLearningClassifier/ClickDLTest.java
index dd1ed6e7..098e6ab8 100644
--- a/src/test/rawDeepLearningClassifier/ClickDLTest.java
+++ b/src/test/rawDeepLearningClassifier/ClickDLTest.java
@@ -1,23 +1,33 @@
package test.rawDeepLearningClassifier;
+import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
+import org.jamdev.jdl4pam.transforms.DLTransform;
import org.jamdev.jdl4pam.transforms.DLTransformsFactory;
import org.jamdev.jdl4pam.transforms.DLTransfromParams;
import org.jamdev.jdl4pam.transforms.SimpleTransformParams;
+import org.jamdev.jdl4pam.transforms.WaveTransform;
+import org.jamdev.jdl4pam.utils.DLMatFile;
import org.jamdev.jpamutils.wavFiles.AudioData;
import org.jamdev.jdl4pam.transforms.DLTransform.DLTransformType;
import org.junit.jupiter.api.Test;
import PamUtils.PamArrayUtils;
+import rawDeepLearningClassifier.dlClassification.animalSpot.StandardModelParams;
+import rawDeepLearningClassifier.dlClassification.archiveModel.ArchiveModelWorker;
import rawDeepLearningClassifier.dlClassification.genericModel.GenericModelParams;
import rawDeepLearningClassifier.dlClassification.genericModel.GenericModelWorker;
import rawDeepLearningClassifier.dlClassification.genericModel.StandardPrediction;
import rawDeepLearningClassifier.segmenter.GroupedRawData;
import us.hebi.matlab.mat.format.Mat5;
import us.hebi.matlab.mat.format.Mat5File;
+import us.hebi.matlab.mat.types.MatFile;
import us.hebi.matlab.mat.types.Matrix;
import us.hebi.matlab.mat.types.Struct;
@@ -26,18 +36,231 @@ import us.hebi.matlab.mat.types.Struct;
* Model from Thomas webber which is a good way to test the click based stuff is working in PAMGUard.
*/
public class ClickDLTest {
-
-
+
+ /**
+ * Test just one click using the zipped classifier
+ * @throws
+ */
@Test
- public void clickDLTest() {
+ public void aclickDLTestZip() {
+
+ System.out.println("*****CLickDLTest: Single click test zip*****");
- float SAMPLE_RATE = 500000;
+ //relative paths to the resource folders.
+ String relModelPath = "/home/jamiemac/Dropbox/PAMGuard_dev/Deep_Learning/click_classifier_Thomas/model_v2/model_pb.zip";
+ String clicksPath = "/home/jamiemac/Dropbox/PAMGuard_dev/Deep_Learning/click_classifier_Thomas/model_v2/example_2000021.mat";
+
+// String matout = "/home/jamiemac/MATLAB-Drive/MATLAB/PAMGUARD/deep_learning/generic_classifier/example_2000021_transforms.mat";
+ String matout=null;
+ // load the click data up.
+ Path clkPath = Paths.get(clicksPath);
+ PredGroupedRawData clickData = null;
+
+ Struct matclkStruct = Mat5.newStruct();
+ try {
+ Mat5File mfr = Mat5.readFromFile(clkPath.toAbsolutePath().normalize().toString());
+
+ // //get array of a name "my_array" from file
+ Struct mlArrayRetrived = mfr.getStruct( "newStruct" );
+
+ Matrix clickWavM = mlArrayRetrived.get("wave", 0);
+
+ double[][] clickWaveform= PamArrayUtils.matrix2array(clickWavM);
+ clickWaveform=PamArrayUtils.transposeMatrix(clickWaveform);
+
+ Matrix clickUID= mlArrayRetrived.get("UID", 0);
+ Matrix pred= mlArrayRetrived.get("pred", 0);
+
+ //create a click object whihc we can pass through transforms etc.
+ clickData = new PredGroupedRawData(0L, 1, 0, clickWaveform[0].length, clickWaveform[0].length);
+ clickData.setUID(clickUID.getLong(0));
+ clickData.setRawData(clickWaveform);
+ clickData.setPrediction(new double[] {pred.getDouble(0)});
+
+ // load the model up
+ Path path = Paths.get(relModelPath);
+
+ ArchiveModelWorker genericModelWorker = new ArchiveModelWorker();
+
+ StandardModelParams genericModelParams = new StandardModelParams();
+ genericModelParams.modelPath = path.toAbsolutePath().normalize().toString();
+
+ //prep the model - all setting are included within the model
+ genericModelWorker.prepModel(genericModelParams, null);
+ System.out.println("seglen: " + genericModelParams.defaultSegmentLen);
+
+ ArrayList<GroupedRawData> groupedData = new ArrayList<GroupedRawData>();
+ groupedData.add(clickData);
+
+ System.out.println("Waveform input: " + groupedData.get(0).getRawData().length + " " + groupedData.get(0).getRawData()[0].length);
+
+ ArrayList<StandardPrediction> genericPrediction = genericModelWorker.runModel(groupedData,96000, 0);
+
+ float[] outputPAMGuard = genericPrediction.get(0).getPrediction();
+
+ System.out.println("Model output PAMGuard: " + outputPAMGuard[0]);
+ assertEquals(outputPAMGuard[0], 0.99, 0.05);
+
+ }
+ catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ assertTrue(false); //make sure the unit test fails
+ return;
+ }
+ }
+
+ /**
+ * Test just one click
+ * @throws
+ */
+ @Test
+ public void aclickDLTest() {
+
+ System.out.println("*****CLickDLTest: Single click test*****");
+
//relative paths to the resource folders.
System.out.println("*****Click classification Deep Learning C*****");
+// //relative paths to the resource folders.
+// String relModelPath = "/Users/jdjm/Library/CloudStorage/Dropbox/PAMGuard_dev/Deep_Learning/click_classifier_Thomas/model_v2/model_pb/saved_model.pb";
+// String clicksPath = "/Users/jdjm/Library/CloudStorage/Dropbox/PAMGuard_dev/Deep_Learning/click_classifier_Thomas/model_v2/example_2000021.mat";
+
//relative paths to the resource folders.
- String relModelPath = "./src/test/resources/rawDeepLearningClassifier/Generic/risso_click/uniform_model/saved_model.pb";
- String clicksPath = "./src/test/resources/rawDeepLearningClassifier/Generic/risso_click/clicks.mat";
+ String relModelPath = "/home/jamiemac/Dropbox/PAMGuard_dev/Deep_Learning/click_classifier_Thomas/model_v2/model_pb/saved_model.pb";
+ String clicksPath = "/home/jamiemac/Dropbox/PAMGuard_dev/Deep_Learning/click_classifier_Thomas/model_v2/example_2000021.mat";
+ //load the click up
+
+// String matout = "/home/jamiemac/MATLAB-Drive/MATLAB/PAMGUARD/deep_learning/generic_classifier/example_2000021_transforms.mat";
+ String matout=null;
+ // load the click data up.
+ Path clkPath = Paths.get(clicksPath);
+ PredGroupedRawData clickData = null;
+
+ Struct matclkStruct = Mat5.newStruct();
+ try {
+ Mat5File mfr = Mat5.readFromFile(clkPath.toAbsolutePath().normalize().toString());
+
+ // //get array of a name "my_array" from file
+ Struct mlArrayRetrived = mfr.getStruct( "newStruct" );
+
+
+ Matrix clickWavM = mlArrayRetrived.get("wave", 0);
+ Matrix modelInputM= mlArrayRetrived.get("wave_pad", 0);
+
+ double[][] clickWaveform= PamArrayUtils.matrix2array(clickWavM);
+ clickWaveform=PamArrayUtils.transposeMatrix(clickWaveform);
+
+ //get the raw model input so we can test the model directly.
+ double[][] pythonModelInput= PamArrayUtils.matrix2array(modelInputM);
+ pythonModelInput = PamArrayUtils.transposeMatrix(pythonModelInput);
+ float[] pythonModelInputF = PamArrayUtils.double2Float(pythonModelInput[0]);
+
+ Matrix clickUID= mlArrayRetrived.get("UID", 0);
+ Matrix pred= mlArrayRetrived.get("pred", 0);
+
+ //create a click object whihc we can pass through transforms etc.
+ clickData = new PredGroupedRawData(0L, 1, 0, clickWaveform[0].length, clickWaveform[0].length);
+ clickData.setUID(clickUID.getLong(0));
+ clickData.setRawData(clickWaveform);
+ clickData.setPrediction(new double[] {pred.getDouble(0)});
+
+
+ // load the model up
+ Path path = Paths.get(relModelPath);
+
+ GenericModelWorker genericModelWorker = new GenericModelWorker();
+
+ GenericModelParams genericModelParams = new GenericModelParams();
+ genericModelParams.modelPath = path.toAbsolutePath().normalize().toString();
+
+
+ //create the transforms.
+ ArrayList<DLTransfromParams> dlTransformParamsArr = new ArrayList<DLTransfromParams>();
+
+ //waveform transforms.
+ dlTransformParamsArr.add(new SimpleTransformParams(DLTransformType.DECIMATE_SCIPY, 96000.));
+ dlTransformParamsArr.add(new SimpleTransformParams(DLTransformType.NORMALISE_WAV, 0., 1, AudioData.ZSCORE)); //needs to be here
+ dlTransformParamsArr.add(new SimpleTransformParams(DLTransformType.PEAK_TRIM, 64, 1));
+
+ genericModelParams.dlTransfromParams = dlTransformParamsArr;
+ genericModelParams.dlTransfroms = DLTransformsFactory.makeDLTransforms((ArrayList<DLTransfromParams>)genericModelParams.dlTransfromParams);
+
+ //create the clicks.
+ path = Paths.get(clicksPath);
+
+ //prep the model
+ genericModelWorker.prepModel(genericModelParams, null);
+
+ ArrayList<GroupedRawData> groupedData = new ArrayList<GroupedRawData>();
+ groupedData.add(clickData);
+
+ System.out.println("Waveform input: " + groupedData.get(0).getRawData().length + " " + groupedData.get(0).getRawData()[0].length);
+
+ ArrayList<StandardPrediction> genericPrediction = genericModelWorker.runModel(groupedData,96000, 0);
+
+// System.out.println("PAMGuard input len: " + pythonModelInputF.length);
+
+ float[] outputPAMGuard = genericPrediction.get(0).getPrediction();
+
+ System.out.println("Model output PAMGuard: " + outputPAMGuard[0]);
+
+ //run the transforms so we can take a look at the inpout
+ ((WaveTransform) genericModelParams.dlTransfroms.get(0)).setWaveData(new AudioData(groupedData.get(0).getRawData()[0], 248000));;
+ //create the transformed wave
+ DLTransform transform = genericModelParams.dlTransfroms.get(0);
+ double[] audioOut = null;
+ for (int i=0; i<genericModelParams.dlTransfroms .size(); i++) {
+ transform = genericModelParams.dlTransfroms.get(i).transformData(transform);
+ audioOut = ((WaveTransform) transform).getWaveData().getScaledSampleAmplitudes();
+ matclkStruct.set(transform.getDLTransformType().getJSONString(), DLMatFile.array2Matrix(audioOut));
+ }
+
+ //RUN THE RAW MODEL with Python transformed input
+
+// System.out.println("Python input len: " + pythonModelInputF.length);
+// float[] outPutPython = genericModelWorker.getModel().runModel(new float[][] {PamArrayUtils.double2Float(audioOut)});
+
+ float[] outPutPython = genericModelWorker.getModel().runModel(new float[][] {pythonModelInputF});
+
+ System.out.println("Model output Python: " + outPutPython[0]);
+
+ assertEquals(outputPAMGuard[0], outPutPython[0], 0.05);
+
+ }
+ catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ assertTrue(false); //make sure the unit test fails
+ return;
+ }
+
+ if (matout!=null) {
+ // Create MAT file with a scalar in a nested struct
+ MatFile matFile = Mat5.newMatFile()
+ .addArray("click_transforms", matclkStruct);
+ // Serialize to disk using default configurations
+ try {
+ Mat5.writeToFile(matFile, matout);
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+ }
+
+
+
+ @Test
+ public void clicksDLTest() {
+
+ float SAMPLE_RATE = 96000;
+ //relative paths to the resource folders.
+ System.out.println("*****CLickDLTest: Clicks test*****");
+
+ //relative paths to the resource folders.
+ String relModelPath = "/home/jamiemac/Dropbox/PAMGuard_dev/Deep_Learning/click_classifier_Thomas/model_v2/model_pb/saved_model.pb";
+ String clicksPath = "/home/jamiemac/Dropbox/PAMGuard_dev/Deep_Learning/click_classifier_Thomas/model_v2/Click_Detector_Click_Detector_Clicks_20220603_111000_classified.mat";
Path path = Paths.get(relModelPath);
@@ -46,59 +269,67 @@ public class ClickDLTest {
GenericModelParams genericModelParams = new GenericModelParams();
genericModelParams.modelPath = path.toAbsolutePath().normalize().toString();
-
+
//create the transforms.
ArrayList<DLTransfromParams> dlTransformParamsArr = new ArrayList<DLTransfromParams>();
//waveform transforms.
- dlTransformParamsArr.add(new SimpleTransformParams(DLTransformType.DECIMATE_SCIPY, 248000.));
- dlTransformParamsArr.add(new SimpleTransformParams(DLTransformType.PEAK_TRIM, 128, 1));
+ dlTransformParamsArr.add(new SimpleTransformParams(DLTransformType.DECIMATE_SCIPY, 96000.));
dlTransformParamsArr.add(new SimpleTransformParams(DLTransformType.NORMALISE_WAV, 0., 1, AudioData.ZSCORE));
-
+ dlTransformParamsArr.add(new SimpleTransformParams(DLTransformType.PEAK_TRIM, 64, 1));
+
genericModelParams.dlTransfromParams = dlTransformParamsArr;
genericModelParams.dlTransfroms = DLTransformsFactory.makeDLTransforms((ArrayList<DLTransfromParams>)genericModelParams.dlTransfromParams);
-
+
//create the clicks.
path = Paths.get(clicksPath);
- ArrayList<PredGroupedRawData> clicks = importClicks(path.toAbsolutePath().normalize().toString(), SAMPLE_RATE);
-
+ ArrayList<PredGroupedRawData> clicks = importClicks(path.toAbsolutePath().normalize().toString(), SAMPLE_RATE);
+
//prep the model
genericModelWorker.prepModel(genericModelParams, null);
-
- System.out.println("Model has loaded");
- ArrayList<GroupedRawData> groupedData = new ArrayList<GroupedRawData>();
-
- for (int i=0; i<1; i++) {
-
+ System.out.println("Model has loaded: n clicks " + clicks.size());
+
+ float count = 0;
+ long timeStart = System.currentTimeMillis();
+ for (int i=0; i<clicks.size(); i++) {
+
float prediction = (float) clicks.get(i).getPrediction()[0];
-
+
+ ArrayList<GroupedRawData> groupedData = new ArrayList<GroupedRawData>();
groupedData.add(clicks.get(i)); //TODO for loop
-
- //System.out.println("Waveform input: " + groupedData.get(i).getRawData().length + " " + groupedData.get(i).getRawData()[0].length);
-
- ArrayList<StandardPrediction> genericPrediction = genericModelWorker.runModel(groupedData,SAMPLE_RATE, 0);
-
- float[] output = genericPrediction.get(i).getPrediction();
-
- System.out.println(String.format("Click %d Predicted output: %.2f true output: %.2f passed: %b", clicks.get(i).getUID(),
- output[0], prediction, output[0]>prediction*0.9 && output[0]<prediction*1.1));
-
- }
-
+ //System.out.println("Waveform input: " + groupedData.get(i).getRawData().length + " " + groupedData.get(i).getRawData()[0].length);
+
+ ArrayList<StandardPrediction> genericPrediction = genericModelWorker.runModel(groupedData,SAMPLE_RATE, 0);
+
+ float[] output = genericPrediction.get(0).getPrediction();
+
+ System.out.println(String.format("Click %d Predicted output: %.4f true output: %.4f passed: %b delta %.2f", clicks.get(i).getUID(),
+ output[0], prediction, output[0]>prediction*0.9 && output[0]<prediction*1.1, (Math.abs(output[0] -prediction))));
+
+ if (output[0]>prediction*0.9 && output[0]<prediction*1.1) {
+ count++;
+ }
+
+ }
+ long timeEnd = System.currentTimeMillis();
+
+ double perctrue = count/clicks.size();
+
+ System.out.println(String.format("Percentage clicks passed: %.2f TIme to process %d clicks - %2f seconds", perctrue, clicks.size(), ((double) (timeEnd-timeStart))/1000.));
}
-
+
/**
* Import a bunch of clicks from a .mat file
*/
public static ArrayList<PredGroupedRawData> importClicks(String filePath, float sR) {
try {
- Mat5File mfr = Mat5.readFromFile(filePath);
+ Mat5File mfr = Mat5.readFromFile(filePath);
// //get array of a name "my_array" from file
- Struct mlArrayRetrived = mfr.getStruct( "clickpreds" );
+ Struct mlArrayRetrived = mfr.getStruct( "binarydata" );
int numClicks= mlArrayRetrived.getNumCols();
ArrayList<PredGroupedRawData> clicks = new ArrayList<PredGroupedRawData>(numClicks);
@@ -106,12 +337,12 @@ public class ClickDLTest {
PredGroupedRawData clickData;
for (int i=0; i<numClicks; i++) {
Matrix clickWav= mlArrayRetrived.get("wave", i);
-
+
double[][] clickwaveform= PamArrayUtils.matrix2array(clickWav);
-
+
clickwaveform = PamArrayUtils.transposeMatrix(clickwaveform);
//System.out.println("click: " + click[0].length + " num: " + numClicks);
-
+
Matrix clickUID= mlArrayRetrived.get("UID", i);
Matrix clickmillis= mlArrayRetrived.get("millis", i);
Matrix channelMap= mlArrayRetrived.get("channelMap", i);
@@ -122,11 +353,11 @@ public class ClickDLTest {
clickData = new PredGroupedRawData(clickmillis.getLong(0), channelMap.getInt(0), startSample.getLong(0), sampleDuration.getLong(0), sampleDuration.getInt(0));
clickData.setUID(clickUID.getLong(0));
clickData.setRawData(clickwaveform);
- clickData.setPrediction(new double[] {pred.getDouble(0), pred.getDouble(1)});
-
+ clickData.setPrediction(new double[] {pred.getDouble(0)});
+
clicks.add(clickData);
}
-
+
return clicks;
}
catch (Exception e) {
@@ -135,11 +366,11 @@ public class ClickDLTest {
return null;
}
}
-
+
public static class PredGroupedRawData extends GroupedRawData {
-
+
private double[] prediction;
-
+
public double[] getPrediction() {
return prediction;
}
@@ -151,10 +382,10 @@ public class ClickDLTest {
public PredGroupedRawData(long timeMilliseconds, int channelBitmap, long startSample, long duration, int samplesize) {
super(timeMilliseconds, channelBitmap, startSample, duration, samplesize);
}
-
-
-
+
+
+
}
-
-
+
+
}