nordic-dev.net/PAMGuard
nordic-dev.net/PAMGuard
2
0
Fork 0
mirror of https://github.com/PAMGuard/PAMGuard.git synced 2024-11-24 16:12:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update pom to use GitHub jpam repository

Browse Source
This commit is contained in:
Jamie Mac 2022-01-13 13:54:06 +00:00
parent 908627b0dd
commit 0a44220fef
8 changed files with 157 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
pom.xml
View File

@ -16,8 +16,8 @@
<properties>
<javafx.version>16</javafx.version>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<maven.compiler.source>16</maven.compiler.source>
<maven.compiler.target>16</maven.compiler.target>
</properties>
<build>
@ -246,6 +246,17 @@
</snapshots>
<url>file://${project.basedir}/repo</url>
</repository>
<!-- Repo for jpam code- this used for deep learning-->
<repository>
<id>jdl4pam</id>
<url>https://raw.github.com/macster110/jpam/mvn-repo/</url>
</repository>
<repository>
<id>jpamutils</id>
<url>https://raw.github.com/macster110/jpam/mvn-repo/</url>
</repository>
<!-- Repo for Jamie's code -->
<!-- IMPORTANT: SOMEHOW THIS REPO INTERFERES WITH THE
@ -255,14 +266,14 @@
repository. Let Maven download the JavaFX files, and
then uncomment this repo again.
Same thing seems to be occurring if I try to update the postgresql library. -->
<repository>
<snapshots>
<enabled>false</enabled>
</snapshots>
<id>central</id>
<name>a0u0ltozdsehx-artifactory-primary-0-releases</name>
<url>https://jmachund.jfrog.io/artifactory/jpam2</url>
</repository>
<!-- <repository> -->
<!-- <snapshots> -->
<!-- <enabled>false</enabled> -->
<!-- </snapshots> -->
<!-- <id>central</id> -->
<!-- <name>a0u0ltozdsehx-artifactory-primary-0-releases</name> -->
<!-- <url>https://jmachund.jfrog.io/artifactory/jpam2</url> -->
<!-- </repository> -->
<!-- Repo for netCDF -->
<repository>
@ -321,7 +332,7 @@
<dependency>
<groupId>org.jamdev</groupId>
<artifactId>jdl4pam</artifactId>
<version>0.0.91</version>
<version>0.0.92</version>
</dependency>
<!-- https://mvnrepository.com/artifact/gov.nist.math/jama -->
@ -660,11 +671,11 @@
<!-- https://mvnrepository.com/artifact/org.springframework/org.springframework.core -->
<!-- https://mvnrepository.com/artifact/org.postgresql/postgresql -->
<dependency>
<!--<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.2.24</version>
</dependency>
</dependency>-->
<!-- https://mvnrepository.com/artifact/org.renjin/renjin-script-engine -->
<dependency>

5
src/binaryFileStorage/BinaryDataSource.java
View File

@ -255,6 +255,11 @@ public abstract class BinaryDataSource {
pamDataUnit.setDataUnitFileInformation(dataUnitFileInformation);
}
}
//sometimes get a null pointer exception here when trying to close PAMGuard?
if (getBinaryStorageStream() == null) {
return false;
}
return getBinaryStorageStream().storeData(data.getObjectType(), pamDataUnit.getBasicData(), data);
}

4
src/rawDeepLearningClassifier/dlClassification/genericModel/DLModelWorker.java
View File

@ -99,11 +99,11 @@ public abstract class DLModelWorker<T> {
}
//run the model. The output is the
//run the model.
float[] output = null;
long time1 = System.currentTimeMillis();
output = runModel(transformedDataStack);
//System.out.println("Out: " + PamArrayUtils.array2String(output, 2, ","));
//System.out.println("Model out: " + PamArrayUtils.array2String(output, 2, ","));
long time2 = System.currentTimeMillis();

3
src/rawDeepLearningClassifier/dlClassification/genericModel/GenericDLClassifier.java
View File

@ -127,8 +127,11 @@ public class GenericDLClassifier implements DLClassiferModel, PamSettings {
*/
if ((PamCalendar.isSoundFile() && !forceQueue) || dlControl.isViewer()) {
//run the model
//System.out.println("GENERICDLCLASSIFIER: Run here!!!");
ArrayList<GenericPrediction> modelResult = getGenericDLWorker().runModel(groupedRawData,
groupedRawData.get(0).getParentDataBlock().getSampleRate(), 0);
//System.out.println("GENERICDLCLASSIFIER: RESULTS!!!: " + modelResult);
if (modelResult==null) {
genericModelWarning.setWarningMessage("Generic deep learning model returned null");

3
src/rawDeepLearningClassifier/dlClassification/genericModel/GenericModelTest.java
View File

@ -18,12 +18,13 @@ public class GenericModelTest {
public static void rightWhaleTest() {
//the model path
String modelPath = "/Users/au671271/Desktop/model_lenet_dropout_input_conv_all/saved_model.pb";
String modelPath = "/Users/au671271/git/PAMGuard_resources/deep_learning/right_whale_tutorial/model_lenet_dropout_input_conv_all/saved_model.pb";
//the audio file to test
String wavFilePath = "/Users/au671271/Google Drive/PAMGuard_dev/Deep_Learning/Right_whales_DG/SouthernRightWhale001-v1/sar98_trk3_8000.wav";
wavFilePath = "/Users/au671271/Google Drive/PAMGuard_dev/Deep_Learning/Right_whales_DG/SouthernRightWhale001-v1/wav_files_timestamp/PAM_20010327_113000.wav";
wavFilePath = "/Users/au671271/git/PAMGuard_resources/deep_learning/right_whale_tutorial/wav/wav_files_timestamp/PAM_20010327_113200.wav";
//define some bits and pieces we need for the classiifer.
float sr = 2000;

1
src/rawDeepLearningClassifier/dlClassification/genericModel/GenericModelWorker.java
View File

@ -24,6 +24,7 @@ public class GenericModelWorker extends DLModelWorker<GenericPrediction> {
@Override
public float[] runModel(float[][][] transformedDataStack) {
//System.out.println("RUN GENERIC MODEL: " + transformedDataStack.length + " " + transformedDataStack[0].length + " " + transformedDataStack[0][0].length);
//System.out.println("RUN GENERIC MODEL: " + transformedDataStack[0][0][0]);
float[] results = genericModel.runModel(transformedDataStack);
//System.out.println("GENERIC MODEL RESULTS: " + results== null ? null : results.length);
return results;

115
src/rawDeepLearningClassifier/dlClassification/ketos/KetosModelTest.java Normal file
View File

@ -0,0 +1,115 @@
package rawDeepLearningClassifier.dlClassification.ketos;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import org.jamdev.jdl4pam.ketos.KetosModel;
import org.jamdev.jdl4pam.ketos.KetosParams;
import org.jamdev.jdl4pam.transforms.DLTransform;
import org.jamdev.jdl4pam.transforms.DLTransformsFactory;
import org.jamdev.jdl4pam.transforms.FreqTransform;
import org.jamdev.jdl4pam.transforms.WaveTransform;
import org.jamdev.jdl4pam.transforms.jsonfile.DLTransformsParser;
import org.jamdev.jdl4pam.utils.DLUtils;
import org.jamdev.jpamutils.wavFiles.AudioData;
/**
* Test the Ketos models in PAMGuard.
* <p>
* Note a similar class also exists in the JPAM repository but this test ensure PAMGuard dependencies are up to date
* etc.
*
* @author Jamie Macaulay
*
*/
public class KetosModelTest {
public static void main(String[] args) {
//test on a right whale.
//File file = new File("/Volumes/GoogleDrive/My Drive/PAMGuard_dev/Deep_Learning/Meridian/right_whales/for_pamguard/narw.ktpb");
File file = new File("/Volumes/GoogleDrive-108005893101854397430/My Drive/PAMGuard_dev/Deep_Learning/Meridian/humpback_whales/SOCAL_Mn_Network.ktpb");
//File file = new File("/Volumes/GoogleDrive-108005893101854397430/My Drive/PAMGuard_dev/Deep_Learning/Meridian/orca/kw_detector_v11_5s.ktpb");
//the wav file to test.
String wavFilePath = "/Volumes/GoogleDrive/My Drive/PAMGuard_dev/Deep_Learning/Meridian/right_whales/for_pamguard/input.wav";
try {
//the ketos model.
KetosModel ketosModel = new KetosModel(file);
//read the JSOn string from the the file.
String jsonString = DLTransformsParser.readJSONString(new File(ketosModel.getAudioReprFile()));
//get the audio representation file.
KetosParams ketosParams = new KetosParams(jsonString);
//System.out.println(ketosParams.toString());
//Open wav files.
AudioData soundData = DLUtils.loadWavFile(wavFilePath);
soundData = soundData.trim(0, (int) (soundData.getSampleRate()*3.0));
//generate the transforms.
ArrayList<DLTransform> transforms = DLTransformsFactory.makeDLTransforms(ketosParams.dlTransforms);
((WaveTransform) transforms.get(0)).setWaveData(soundData);
DLTransform transform = transforms.get(0);
for (int i=0; i<transforms.size(); i++) {
// System.out.println(transforms);
transform = transforms.get(i).transformData(transform);
// if (i==1) {
// transfromedData = DLMatFile.array2Matrix(((FreqTransform) transform).getSpecTransfrom().getTransformedData());
// }
// if (transforms.get(i) instanceof FreqTransform) {
// transformedData = ((FreqTransform) transform).getSpecTransfrom().getTransformedData();
// System.out.println("Transform shape: " + i + " " + transformedData.length + " " + transformedData[0].length);
// }
}
double[][] transformedData = ((FreqTransform) transform).getSpecTransfrom().getTransformedData();
float[] output = null;
float[][][] data;
int nStack = 1; //number of specs to give to the classifier.
for (int i=0; i<10; i++) {
long time1 = System.currentTimeMillis();
data = new float[nStack][][];
for (int j=0; j<nStack; j++) {
data[j] = DLUtils.toFloatArray(transformedData);
}
output = ketosModel.runModel(data);
long time2 = System.currentTimeMillis();
System.out.println("Time to run model: " + (time2-time1) + " ms");
}
double[] prob = new double[output.length];
for (int j=0; j<output.length; j++) {
//python code for this.
// prob = torch.nn.functional.softmax(out).numpy()[n, 1]
// pred = int(prob >= ARGS.threshold)
//softmax function
System.out.println("The output is: " + output[j]);
prob[j] = DLUtils.softmax(output[j], output);
System.out.println("The probability is: " + prob[j]);
}
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}

6
src/rawDeepLearningClassifier/layoutFX/dlTransfroms/DLTransformsPane.java
View File

@ -6,6 +6,7 @@ import java.util.List;
import org.jamdev.jdl4pam.transforms.DLTransform;
import org.jamdev.jdl4pam.transforms.DLTransform.DLTransformType;
import org.jamdev.jdl4pam.transforms.FreqTransform;
import org.jamdev.jdl4pam.transforms.SimpleTransform;
import org.jamdev.jdl4pam.transforms.WaveTransform;
import javafx.geometry.Pos;
@ -149,6 +150,10 @@ public class DLTransformsPane extends PamBorderPane {
ArrayList<DLTransformPane> dlTransformPanes = new ArrayList<DLTransformPane>();
sampleRate=-1;
// for (int i=0; i<dlTransforms.size() ; i++) {
// System.out.println("Set DL transfroms: " + dlTransforms.get(i).getDLTransformType() + " " + ((SimpleTransform) dlTransforms.get(i)).getParams());
// }
//create a pane for each transform
for (int i=0; i<dlTransforms.size() ; i++) {
@ -157,7 +162,6 @@ public class DLTransformsPane extends PamBorderPane {
sampleRate = ((WaveTransform) dlTransforms.get(i)).getWaveData().getSampleRate();
}
// System.out.println("Set DL transfroms: " + dlTransforms.get(i).getDLTransformType());
dlTransformPanes.add(DataTransformPaneFactory.getSettingsPane(dlTransforms.get(i)));
dlTransformPanes.get(i).setParams(dlTransforms.get(i));
dlTransformPanes.get(i).addSettingsListener(()->{