diff --git a/src/cpod/CPODClickDataBlock.java b/src/cpod/CPODClickDataBlock.java
index 05feedd8..f25b8d33 100644
--- a/src/cpod/CPODClickDataBlock.java
+++ b/src/cpod/CPODClickDataBlock.java
@@ -1,6 +1,5 @@
package cpod;
-import PamController.PamControlledUnit;
import PamguardMVC.AcousticDataBlock;
import PamguardMVC.PamProcess;
import cpod.dataSelector.CPODDataSelectorCreator;
@@ -30,32 +29,22 @@ public class CPODClickDataBlock extends AcousticDataBlock<CPODClick> {
}
}
- /* (non-Javadoc)
- * @see PamguardMVC.PamDataBlock#getDataName()
- */
- @Override
- public String getDataName() {
- if (getParentProcess()!=null) {
- PamControlledUnit cpodControl = getParentProcess().getPamControlledUnit();
- if (cpodControl == null) {
- return super.getDataName();
- }
- return cpodControl.getUnitName() + "_" + getDataTypeString();
- }
- return "CPOD_temporaty";
-
- }
+// /* (non-Javadoc)
+// * @see PamguardMVC.PamDataBlock#getDataName()
+// */
+// @Override
+// public String getDataName() {
+// if (getParentProcess()!=null) {
+// PamControlledUnit cpodControl = getParentProcess().getPamControlledUnit();
+// if (cpodControl == null) {
+// return super.getDataName();
+// }
+// return cpodControl.getUnitName() + "_" + getDataTypeString();
+// }
+// return "CPOD_temporary";
+//
+// }
- private String getDataTypeString() {
- switch (clikcType) {
- case CPODMap.FILE_CP1:
- return "CP1";
- case CPODMap.FILE_CP3:
- return "CP3";
- }
- return null;
- }
-
@Override
public float getSampleRate() {
diff --git a/src/group3dlocaliser/offline/Group3DOfflineTask.java b/src/group3dlocaliser/offline/Group3DOfflineTask.java
index 71b5efa9..ab525a7b 100644
--- a/src/group3dlocaliser/offline/Group3DOfflineTask.java
+++ b/src/group3dlocaliser/offline/Group3DOfflineTask.java
@@ -31,7 +31,12 @@ public class Group3DOfflineTask extends OfflineTask<PamDataUnit>{
if (parentData != null) {
this.addRequiredDataBlock(parentData);
}
+
+ //here be large bugs in all data and select data processing if not included
this.addRequiredDataBlock(ArrayManager.getArrayManager().getHydrophoneDataBlock());
+ this.addRequiredDataBlock(ArrayManager.getArrayManager().getStreamerDatabBlock());
+ this.addRequiredDataBlock(ArrayManager.getGPSDataBlock());
+
}
@Override
diff --git a/src/rawDeepLearningClassifier/DLControl.java b/src/rawDeepLearningClassifier/DLControl.java
index 4ea85db9..06663a8a 100644
--- a/src/rawDeepLearningClassifier/DLControl.java
+++ b/src/rawDeepLearningClassifier/DLControl.java
@@ -34,7 +34,7 @@ import rawDeepLearningClassifier.dlClassification.DLClassiferModel;
import rawDeepLearningClassifier.dlClassification.DLClassifierChooser;
import rawDeepLearningClassifier.dlClassification.DLClassifyProcess;
import rawDeepLearningClassifier.dlClassification.animalSpot.SoundSpotClassifier;
-import rawDeepLearningClassifier.dlClassification.archiveModel.ZipModelClassifier;
+import rawDeepLearningClassifier.dlClassification.archiveModel.PamZipModelClassifier;
import rawDeepLearningClassifier.dlClassification.genericModel.GenericDLClassifier;
import rawDeepLearningClassifier.dlClassification.ketos.KetosClassifier2;
import rawDeepLearningClassifier.dlClassification.koogu.KooguClassifier;
@@ -256,7 +256,8 @@ public class DLControl extends PamControlledUnit implements PamSettings {
dlModels.add(new SoundSpotClassifier(this));
dlModels.add(new KetosClassifier2(this));
dlModels.add(new KooguClassifier(this));
- dlModels.add(new ZipModelClassifier(this));
+ dlModels.add(new PamZipModelClassifier(this));
+
//it is important the Generic Model is last because we need to check
//for PG metadata in all other models before resorting to manually
//setting up a model.
diff --git a/src/rawDeepLearningClassifier/DLStatus.java b/src/rawDeepLearningClassifier/DLStatus.java
index f57058ef..d23e4a0c 100644
--- a/src/rawDeepLearningClassifier/DLStatus.java
+++ b/src/rawDeepLearningClassifier/DLStatus.java
@@ -36,7 +36,9 @@ public enum DLStatus {
" There are no prediction classes selected for classification. Predicitons for each segment will be saved but there will be no detections generated",
ErrorLevel.WARNING),
- DECOMPRESSING_MODEL("Decompressing model", "Decompressing the model file", ErrorLevel.NO_ERROR), ;
+ DECOMPRESSING_MODEL("Decompressing model", "Decompressing the model file", ErrorLevel.NO_ERROR),
+
+ INCOMPATIBLE_ZIP("Incorrect Zip format", "The zip format is incorrect. The zip file should have a *.pgdl file in the parent directory along with either a Tensorflow or PyTorch model.", ErrorLevel.ERROR), ;
/**
diff --git a/src/rawDeepLearningClassifier/dlClassification/DLClassiferModel.java b/src/rawDeepLearningClassifier/dlClassification/DLClassiferModel.java
index c86efd60..38fd9e9c 100644
--- a/src/rawDeepLearningClassifier/dlClassification/DLClassiferModel.java
+++ b/src/rawDeepLearningClassifier/dlClassification/DLClassiferModel.java
@@ -42,8 +42,9 @@ public interface DLClassiferModel {
/**
* Called whenever PAMGuard stops.
+ * @return
*/
- public void setModel(URI model);
+ public DLStatus setModel(URI model);
/**
* Check whether a URI is compatible with a classification framework
diff --git a/src/rawDeepLearningClassifier/dlClassification/StandardClassifierModel.java b/src/rawDeepLearningClassifier/dlClassification/StandardClassifierModel.java
index 8bde670a..f9992cb7 100644
--- a/src/rawDeepLearningClassifier/dlClassification/StandardClassifierModel.java
+++ b/src/rawDeepLearningClassifier/dlClassification/StandardClassifierModel.java
@@ -101,7 +101,7 @@ public abstract class StandardClassifierModel implements DLClassiferModel, PamSe
@Override
public void prepModel() {
// System.out.println("STANDARD CLASSIFIER MODEL PREP MODEL! !!!");
- StandardModelParams oldParams = getDLParams().clone();
+// StandardModelParams oldParams = getDLParams().clone();
getDLWorker().prepModel(getDLParams(), dlControl);
@@ -174,17 +174,15 @@ public abstract class StandardClassifierModel implements DLClassiferModel, PamSe
// + "Predicitons for each segment will be saved but there will be no detections generated",
// 1));
}
-
-
-
return DLStatus.MODEL_LOAD_SUCCESS;
}
@Override
- public void setModel(URI uri) {
+ public DLStatus setModel(URI uri) {
//will change the params if we do not clone.
StandardModelParams.setModel(uri, this.getDLParams());
this.prepModel();
+ return getModelStatus();
}
diff --git a/src/rawDeepLearningClassifier/dlClassification/archiveModel/ArchiveModelUI.java b/src/rawDeepLearningClassifier/dlClassification/archiveModel/ArchiveModelUI.java
index 7bb2a59b..73c6a21d 100644
--- a/src/rawDeepLearningClassifier/dlClassification/archiveModel/ArchiveModelUI.java
+++ b/src/rawDeepLearningClassifier/dlClassification/archiveModel/ArchiveModelUI.java
@@ -28,6 +28,8 @@ public class ArchiveModelUI implements DLCLassiferModelUI {
*/
private ArrayList<ExtensionFilter> extensionFilters;
+
+
/**
* SondSpot classifier.
* @param soundSpotClassifier
@@ -79,6 +81,14 @@ public class ArchiveModelUI implements DLCLassiferModelUI {
}
return extensionFilters;
}
+
+ /**
+ * Set the extension filters.
+ * @param extensionFilters
+ */
+ public void setExtensionFilters(ArrayList<ExtensionFilter> extensionFilters) {
+ this.extensionFilters = extensionFilters;
+ }
diff --git a/src/rawDeepLearningClassifier/dlClassification/archiveModel/ArchiveModelWorker.java b/src/rawDeepLearningClassifier/dlClassification/archiveModel/ArchiveModelWorker.java
index d5f17acf..e694c9e9 100644
--- a/src/rawDeepLearningClassifier/dlClassification/archiveModel/ArchiveModelWorker.java
+++ b/src/rawDeepLearningClassifier/dlClassification/archiveModel/ArchiveModelWorker.java
@@ -2,8 +2,13 @@ package rawDeepLearningClassifier.dlClassification.archiveModel;
import java.io.File;
import java.io.IOException;
+import java.net.URI;
import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipException;
+import java.util.zip.ZipFile;
import org.jamdev.jdl4pam.ArchiveModel;
import org.jamdev.jdl4pam.genericmodel.GenericModelParams;
@@ -190,6 +195,13 @@ public class ArchiveModelWorker extends DLModelWorker<GenericPrediction> {
* @throws IOException
*/
public ArchiveModel loadModel(String currentPath2) throws MalformedModelException, IOException {
+ //note the the model should have been check for compatibility beforehand
+ File file = new File(currentPath2);
+
+ String model = getZipFilePath(file, ".py");
+ if (model==null) model = getZipFilePath(file, ".pb");
+ String settings = getZipFilePath(file, ".pdtf");
+
return new SimpleArchiveModel(new File(currentPath2));
}
@@ -248,6 +260,33 @@ public class ArchiveModelWorker extends DLModelWorker<GenericPrediction> {
public boolean isModelNull() {
return dlModel==null;
}
+
+ /**
+ * Find the first file within a zip folder that matches a pattern.
+ * @param zipFile - uri to the zip file
+ * @param filePattern - the file pattern to match - the file must contain this string.
+ * @return null if no file found and the file pqth if the file is founf
+ * @throws ZipException
+ * @throws IOException
+ */
+ static String getZipFilePath(File zipFileIn, String filePattern) throws ZipException, IOException {
+
+ try (ZipFile zipFile = new ZipFile(zipFileIn)) {
+ Enumeration<? extends ZipEntry> entries = zipFile.entries();
+ //this iterates through all files, including in sub folders.
+ while (entries.hasMoreElements()) {
+ ZipEntry entry = entries.nextElement();
+ // Check if entry is a directory
+ if (!entry.isDirectory()) {
+ //System.out.println(entry);
+ if (entry.getName().contains(filePattern)) {
+ return entry.getName();
+ }
+ }
+ }
+ }
+ return null;
+ }
}
\ No newline at end of file
diff --git a/src/rawDeepLearningClassifier/dlClassification/archiveModel/PamZipModelClassifier.java b/src/rawDeepLearningClassifier/dlClassification/archiveModel/PamZipModelClassifier.java
new file mode 100644
index 00000000..5537fb0b
--- /dev/null
+++ b/src/rawDeepLearningClassifier/dlClassification/archiveModel/PamZipModelClassifier.java
@@ -0,0 +1,106 @@
+package rawDeepLearningClassifier.dlClassification.archiveModel;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.util.zip.ZipException;
+import rawDeepLearningClassifier.DLControl;
+import rawDeepLearningClassifier.DLStatus;
+
+/**
+ * Loads a zip file and checks for a saved model alongside a pamguard settings file
+ * <p>
+ * The model quickly pre checks zip files to make sure there is a settings file inside.
+ * This means that non compatible zip files are greyed out in the file importer.
+ * <p>
+ * The model can accept PyTorch or Tensorflow models.
+ *
+ * @author Jamie Macaulay
+ *
+ */
+public class PamZipModelClassifier extends ArchiveModelClassifier {
+
+ private static final String MODEL_NAME = "PAMGuard model";
+
+ /**
+ * The file extensions
+ */
+ private String[] fileExtensions = new String[] {"*.zip"};
+
+ public PamZipModelClassifier(DLControl dlControl) {
+ super(dlControl);
+ }
+
+ @Override
+ public String getName() {
+ return MODEL_NAME;
+ }
+
+
+ @Override
+ public DLStatus setModel(URI zipUri) {
+ //will change the params if we do not clone.
+ //first check whether the zip file has the correct model.
+// ZipUtils.
+
+ //check that we have some kind of model that we can load here. Do not attempt to load the model if not.
+ String model;
+ try {
+ model = getZipFilePath( zipUri, ".py");
+
+ if (model==null) model = getZipFilePath( zipUri, ".pb");
+ if (model==null) return DLStatus.INCOMPATIBLE_ZIP;
+
+ String settingsFile = getZipFilePath( zipUri, ".pdtf");
+ if (settingsFile==null) return DLStatus.INCOMPATIBLE_ZIP;
+ } catch (ZipException e) {
+ e.printStackTrace();
+ return DLStatus.INCOMPATIBLE_ZIP;
+ } catch (IOException e) {
+ e.printStackTrace();
+ return DLStatus.INCOMPATIBLE_ZIP;
+ }
+
+ return super.setModel(zipUri);
+ }
+
+ /**
+ * Find the first file within a zip folder that matches a pattern.
+ * @param zipUri - uri to the zip file
+ * @param filePattern - the file pattern to match - the file must contain this string.
+ * @return null if no file found and the file pqth if the file is founf
+ * @throws ZipException
+ * @throws IOException
+ */
+ private static String getZipFilePath(URI zipUri, String filePattern) throws ZipException, IOException {
+ return ArchiveModelWorker.getZipFilePath(new File(zipUri), filePattern);
+ }
+
+ /**
+ * Zip test.
+ * @param args
+ */
+ public static void main(String[] args) {
+
+ String fileName = "D:/Dropbox/PAMGuard_dev/Deep_Learning/Gibbons/gibbon_model_shared/gibbon_model.zip";
+
+ URI zipUri = new File(fileName).toURI();
+
+ try {
+ String pbFile = getZipFilePath( zipUri, ".pb");
+ String transformFile = getZipFilePath( zipUri, ".pdtf");
+
+ System.out.println("pbFile: " + pbFile + " transformFile: " + transformFile);
+
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+
+
+
+
+}
+
+
diff --git a/src/rawDeepLearningClassifier/dlClassification/archiveModel/SimpleArchiveModel.java b/src/rawDeepLearningClassifier/dlClassification/archiveModel/SimpleArchiveModel.java
index eb7e8b4d..db7afaaf 100644
--- a/src/rawDeepLearningClassifier/dlClassification/archiveModel/SimpleArchiveModel.java
+++ b/src/rawDeepLearningClassifier/dlClassification/archiveModel/SimpleArchiveModel.java
@@ -2,6 +2,12 @@ package rawDeepLearningClassifier.dlClassification.archiveModel;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.jamdev.jdl4pam.ArchiveModel;
@@ -14,21 +20,37 @@ import ai.djl.MalformedModelException;
*/
public class SimpleArchiveModel extends ArchiveModel {
+
public SimpleArchiveModel(File file) throws MalformedModelException, IOException {
super(file);
- // TODO Auto-generated constructor stub
}
@Override
public String getAudioReprRelPath(String zipFolder) {
- // settings are in parent directory
- return "audio_repr_pg.json";
+ try {
+ System.out.println("SETTINGS PATH: " + getRelFilePath(zipFolder, ".pdtf"));
+ return getRelFilePath(zipFolder, ".pdtf");
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ return null;
+ }
}
@Override
public String getModelRelPath(String zipFolder) {
- // model is in parent directory
- return "model/saved_model.pb";
+ try {
+ String model = null;
+ model = getRelFilePath(zipFolder, ".pb");
+ if (model==null) model = getRelFilePath(zipFolder, ".py");
+ System.out.println("MODEL PATH: " +model);
+ return model;
+
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ return null;
+ }
}
@Override
@@ -36,4 +58,28 @@ public class SimpleArchiveModel extends ArchiveModel {
return "zip_model";
}
+
+ private static String getRelFilePath(String zipFolder, String fileEnd) throws IOException {
+ // find files matched `png` file extension from folder C:\\test
+ try (Stream<Path> walk = Files.walk(Paths.get(zipFolder))) {
+ List<String> result = walk
+ .filter(p -> !Files.isDirectory(p)) // not a directory
+ .map(p -> p.toString().toLowerCase()) // convert path to string
+ .filter(f -> f.endsWith(fileEnd)) // check end with
+ .collect(Collectors.toList()); // collect all matched to a List
+
+ if (result.size()>0) {
+ String firstFile = result.get(0);
+
+ String relative = new File(zipFolder).toURI().relativize(new File(firstFile).toURI()).getPath();
+
+ return relative;
+ }
+ else return null;
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ return null;
+ }
+ }
}
diff --git a/src/rawDeepLearningClassifier/dlClassification/archiveModel/ZipModelClassifier.java b/src/rawDeepLearningClassifier/dlClassification/archiveModel/ZipModelClassifier.java
deleted file mode 100644
index 57875c2f..00000000
--- a/src/rawDeepLearningClassifier/dlClassification/archiveModel/ZipModelClassifier.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package rawDeepLearningClassifier.dlClassification.archiveModel;
-
-import rawDeepLearningClassifier.DLControl;
-
-public class ZipModelClassifier extends ArchiveModelClassifier {
-
-
- public static String MODEL_NAME = "Zip";
-
- /**
- * The file extensions
- */
- private String[] fileExtensions = new String[] {"zip"};
-
-
- public ZipModelClassifier(DLControl dlControl) {
- super(dlControl);
- }
-
- @Override
- public String[] getFileExtensions() {
- return fileExtensions;
- }
-
- @Override
- public String getName() {
- return MODEL_NAME;
- }
-}
diff --git a/src/rawDeepLearningClassifier/dlClassification/dummyClassifier/DummyClassifier.java b/src/rawDeepLearningClassifier/dlClassification/dummyClassifier/DummyClassifier.java
index de0ed73c..25e98ac8 100644
--- a/src/rawDeepLearningClassifier/dlClassification/dummyClassifier/DummyClassifier.java
+++ b/src/rawDeepLearningClassifier/dlClassification/dummyClassifier/DummyClassifier.java
@@ -94,9 +94,9 @@ public class DummyClassifier implements DLClassiferModel{
}
@Override
- public void setModel(URI model) {
+ public DLStatus setModel(URI model) {
// TODO Auto-generated method stub
-
+ return null;
}
}
diff --git a/src/rawDeepLearningClassifier/dlClassification/koogu/KooguModelWorker.java b/src/rawDeepLearningClassifier/dlClassification/koogu/KooguModelWorker.java
index 108a41f8..4026575a 100644
--- a/src/rawDeepLearningClassifier/dlClassification/koogu/KooguModelWorker.java
+++ b/src/rawDeepLearningClassifier/dlClassification/koogu/KooguModelWorker.java
@@ -9,6 +9,12 @@ import org.jamdev.jdl4pam.koogu.KooguModel;
import ai.djl.MalformedModelException;
import rawDeepLearningClassifier.dlClassification.archiveModel.ArchiveModelWorker;
+/**
+ * The Koogu model worker.
+ *
+ * @author Jamie Macaulay
+ *
+ */
public class KooguModelWorker extends ArchiveModelWorker {
diff --git a/src/rawDeepLearningClassifier/dlClassification/orcaSpot/OrcaSpotClassifier.java b/src/rawDeepLearningClassifier/dlClassification/orcaSpot/OrcaSpotClassifier.java
index aeac76e6..21ff1bb0 100644
--- a/src/rawDeepLearningClassifier/dlClassification/orcaSpot/OrcaSpotClassifier.java
+++ b/src/rawDeepLearningClassifier/dlClassification/orcaSpot/OrcaSpotClassifier.java
@@ -358,9 +358,9 @@ public class OrcaSpotClassifier implements DLClassiferModel, PamSettings {
}
@Override
- public void setModel(URI model) {
+ public DLStatus setModel(URI model) {
// TODO Auto-generated method stub
-
+ return null;
}
}
diff --git a/src/rawDeepLearningClassifier/layoutFX/DLModelSelectPane.java b/src/rawDeepLearningClassifier/layoutFX/DLModelSelectPane.java
index 3b03d5de..2d6a8e22 100644
--- a/src/rawDeepLearningClassifier/layoutFX/DLModelSelectPane.java
+++ b/src/rawDeepLearningClassifier/layoutFX/DLModelSelectPane.java
@@ -375,12 +375,12 @@ public class DLModelSelectPane extends PamBorderPane {
try {
//we are loading model from a file - anything can happen so put in a try catch.
- currentClassifierModel.setModel(file);
-
- if (currentClassifierModel.getModelStatus().isError()) {
+ DLStatus status = currentClassifierModel.setModel(file);
+
+ if (status.isError()) {
System.err.println("Model load failed: " + currentClassifierModel.getModelStatus());
currentClassifierModel=null;
- return DLStatus.MODEL_LOAD_FAILED;
+ return status;
}
return DLStatus.MODEL_LOAD_SUCCESS;