Bug fixes to CPOD imports and implementing a zip model in PAMGuard.

src/cpod/CPODClickDataBlock.java

@@ -1,6 +1,5 @@
 package cpod;
 
-import PamController.PamControlledUnit;
 import PamguardMVC.AcousticDataBlock;
 import PamguardMVC.PamProcess;
 import cpod.dataSelector.CPODDataSelectorCreator;
@@ -30,32 +29,22 @@ public class CPODClickDataBlock extends AcousticDataBlock<CPODClick> {
 		}
 	}
 
-	/* (non-Javadoc)
-	 * @see PamguardMVC.PamDataBlock#getDataName()
-	 */
-	@Override
-	public String getDataName() {
-		if (getParentProcess()!=null) {
-			PamControlledUnit cpodControl = getParentProcess().getPamControlledUnit();
-			if (cpodControl == null) {
-				return super.getDataName();
-			}
-			return cpodControl.getUnitName() + "_" + getDataTypeString();
-		}
-		return "CPOD_temporaty";
-	
-	}
+//	/* (non-Javadoc)
+//	 * @see PamguardMVC.PamDataBlock#getDataName()
+//	 */
+//	@Override
+//	public String getDataName() {
+//		if (getParentProcess()!=null) {
+//			PamControlledUnit cpodControl = getParentProcess().getPamControlledUnit();
+//			if (cpodControl == null) {
+//				return super.getDataName();
+//			}
+//			return cpodControl.getUnitName() + "_" + getDataTypeString();
+//		}
+//		return "CPOD_temporary";
+//	
+//	}
 
-	private String getDataTypeString() {
-		switch (clikcType) {
-		case CPODMap.FILE_CP1:
-			return "CP1";
-		case CPODMap.FILE_CP3:
-			return "CP3";
-		}
-		return null;
-	}
-	
 	
 	@Override
 	public float getSampleRate() {

src/group3dlocaliser/offline/Group3DOfflineTask.java

@@ -31,7 +31,12 @@ public class Group3DOfflineTask extends OfflineTask<PamDataUnit>{
 		if (parentData != null) {
 			this.addRequiredDataBlock(parentData);
 		}
+		
+		//here be large bugs in all data and select data processing if not included
 		this.addRequiredDataBlock(ArrayManager.getArrayManager().getHydrophoneDataBlock());
+		this.addRequiredDataBlock(ArrayManager.getArrayManager().getStreamerDatabBlock());
+		this.addRequiredDataBlock(ArrayManager.getGPSDataBlock());
+
 	}
 
 	@Override

src/rawDeepLearningClassifier/DLControl.java

@@ -34,7 +34,7 @@ import rawDeepLearningClassifier.dlClassification.DLClassiferModel;
 import rawDeepLearningClassifier.dlClassification.DLClassifierChooser;
 import rawDeepLearningClassifier.dlClassification.DLClassifyProcess;
 import rawDeepLearningClassifier.dlClassification.animalSpot.SoundSpotClassifier;
-import rawDeepLearningClassifier.dlClassification.archiveModel.ZipModelClassifier;
+import rawDeepLearningClassifier.dlClassification.archiveModel.PamZipModelClassifier;
 import rawDeepLearningClassifier.dlClassification.genericModel.GenericDLClassifier;
 import rawDeepLearningClassifier.dlClassification.ketos.KetosClassifier2;
 import rawDeepLearningClassifier.dlClassification.koogu.KooguClassifier;
@@ -256,7 +256,8 @@ public class DLControl extends PamControlledUnit implements PamSettings {
 		dlModels.add(new SoundSpotClassifier(this));
 		dlModels.add(new KetosClassifier2(this));
 		dlModels.add(new KooguClassifier(this));
-		dlModels.add(new ZipModelClassifier(this));
+		dlModels.add(new PamZipModelClassifier(this));
+		
 		//it is important the Generic Model is last because we need to check 
 		//for PG metadata in all other models before resorting to manually 
 		//setting up a model. 

src/rawDeepLearningClassifier/DLStatus.java

@@ -36,7 +36,9 @@ public enum DLStatus {
 			" There are no prediction classes selected for classification. Predicitons for each segment will be saved but there will be no detections generated",
 			ErrorLevel.WARNING), 
 	
-	DECOMPRESSING_MODEL("Decompressing model", "Decompressing the model file", ErrorLevel.NO_ERROR), ;
+	DECOMPRESSING_MODEL("Decompressing model", "Decompressing the model file", ErrorLevel.NO_ERROR), 
+
+	INCOMPATIBLE_ZIP("Incorrect Zip format", "The zip format is incorrect. The zip file should have a *.pgdl file in the parent directory along with either a Tensorflow or PyTorch model.", ErrorLevel.ERROR), ;
 
 
 	/**

src/rawDeepLearningClassifier/dlClassification/DLClassiferModel.java

@@ -42,8 +42,9 @@ public interface DLClassiferModel {
 	
 	/**
 	 * Called whenever PAMGuard stops.
+	 * @return 
 	 */
-	public void setModel(URI model);
+	public DLStatus setModel(URI model);
 	
 	/**
 	 * Check whether a URI is compatible with a classification framework 

src/rawDeepLearningClassifier/dlClassification/StandardClassifierModel.java

@@ -101,7 +101,7 @@ public abstract class StandardClassifierModel implements DLClassiferModel, PamSe
 	@Override
 	public void prepModel() {
 //		System.out.println("STANDARD CLASSIFIER MODEL PREP MODEL! !!!");
-		StandardModelParams oldParams = getDLParams().clone();
+//		StandardModelParams oldParams = getDLParams().clone();
 		
 		getDLWorker().prepModel(getDLParams(), dlControl);
 
@@ -174,17 +174,15 @@ public abstract class StandardClassifierModel implements DLClassiferModel, PamSe
 //							+ "Predicitons for each segment will be saved but there will be no detections generated",
 //					1));
 		}
-
-
-		
 		return DLStatus.MODEL_LOAD_SUCCESS;
 	}
 	
 	@Override
-	public void setModel(URI uri) {
+	public DLStatus setModel(URI uri) {
 		//will change the params if we do not clone. 
 		StandardModelParams.setModel(uri, this.getDLParams()); 
 		this.prepModel();
+		return getModelStatus();
 	}
 	
 	

src/rawDeepLearningClassifier/dlClassification/archiveModel/ArchiveModelUI.java

@@ -28,6 +28,8 @@ public class ArchiveModelUI  implements DLCLassiferModelUI {
 	 */
 	private ArrayList<ExtensionFilter> extensionFilters;
 
+
+
 	/**
 	 * SondSpot classifier. 
 	 * @param soundSpotClassifier
@@ -79,6 +81,14 @@ public class ArchiveModelUI  implements DLCLassiferModelUI {
 		} 
 		return extensionFilters;
 	}
+	
+	/**
+	 * Set the extension filters. 
+	 * @param extensionFilters
+	 */
+	public void setExtensionFilters(ArrayList<ExtensionFilter> extensionFilters) {
+		this.extensionFilters = extensionFilters;
+	}
 
 
 

src/rawDeepLearningClassifier/dlClassification/archiveModel/ArchiveModelWorker.java

@@ -2,8 +2,13 @@ package rawDeepLearningClassifier.dlClassification.archiveModel;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipException;
+import java.util.zip.ZipFile;
 
 import org.jamdev.jdl4pam.ArchiveModel;
 import org.jamdev.jdl4pam.genericmodel.GenericModelParams;
@@ -190,6 +195,13 @@ public class ArchiveModelWorker extends DLModelWorker<GenericPrediction> {
 	 * @throws IOException
 	 */
 	public ArchiveModel loadModel(String currentPath2) throws MalformedModelException, IOException {
+		//note the the model should have been check for compatibility beforehand
+		File file = new File(currentPath2);
+		
+		String model = getZipFilePath(file, ".py");  
+		if (model==null) model = getZipFilePath(file, ".pb"); 
+		String settings = getZipFilePath(file, ".pdtf");  
+
 		return new SimpleArchiveModel(new File(currentPath2)); 
 	}
 
@@ -248,6 +260,33 @@ public class ArchiveModelWorker extends DLModelWorker<GenericPrediction> {
 	public boolean isModelNull() {
 		return dlModel==null;
 	}
+	
+	/**
+	 * Find the first file within a zip folder that matches a pattern. 
+	 * @param zipFile - uri to the zip file
+	 * @param filePattern - the file pattern to match - the file must contain this string. 
+	 * @return null if no file found and the file pqth if the file is founf
+	 * @throws ZipException
+	 * @throws IOException
+	 */
+	static String getZipFilePath(File zipFileIn, String filePattern) throws ZipException, IOException {
+			
+		try (ZipFile zipFile = new ZipFile(zipFileIn)) {
+		    Enumeration<? extends ZipEntry> entries = zipFile.entries();
+		    //this iterates through all files, including in sub folders. 
+		    while (entries.hasMoreElements()) {
+		        ZipEntry entry = entries.nextElement();
+		        // Check if entry is a directory
+		        if (!entry.isDirectory()) {
+		           //System.out.println(entry); 
+		           if (entry.getName().contains(filePattern)) {
+		        	   return entry.getName();
+		           }
+		        }
+		    }
+		}
+		return null;
+	}
 
 
 }

src/rawDeepLearningClassifier/dlClassification/archiveModel/PamZipModelClassifier.java

@@ -0,0 +1,106 @@
+package rawDeepLearningClassifier.dlClassification.archiveModel;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.util.zip.ZipException;
+import rawDeepLearningClassifier.DLControl;
+import rawDeepLearningClassifier.DLStatus;
+
+/**
+ * Loads a zip file and checks for a saved model alongside a pamguard settings file
+ * <p>
+ * The model quickly pre checks zip files to make sure there is a settings file inside. 
+ * This means that non compatible zip files are greyed out in the file importer. 
+ * <p>
+ * The model can accept PyTorch or Tensorflow models. 
+ * 
+ * @author Jamie Macaulay
+ *
+ */
+public class PamZipModelClassifier extends ArchiveModelClassifier {
+
+	private static final String MODEL_NAME = "PAMGuard model";
+	
+	/**
+	 * The file extensions
+	 */
+	private String[] fileExtensions = new String[] {"*.zip"};
+
+	public PamZipModelClassifier(DLControl dlControl) {
+		super(dlControl);
+	}
+	
+	@Override
+	public String getName() {
+		return MODEL_NAME;
+	}
+	
+	
+	@Override
+	public DLStatus setModel(URI zipUri) {
+		//will change the params if we do not clone. 
+		//first check whether the zip file has the correct model. 
+//		ZipUtils.
+		
+		//check that we have some kind of model that we can load here. Do not attempt to load the model if not. 
+		String model;
+		try {
+			model = getZipFilePath( zipUri, ".py");
+
+			if (model==null) model = getZipFilePath( zipUri, ".pb");
+			if (model==null) return DLStatus.INCOMPATIBLE_ZIP;
+
+			String settingsFile = getZipFilePath( zipUri, ".pdtf");
+			if (settingsFile==null)  return DLStatus.INCOMPATIBLE_ZIP;
+		} catch (ZipException e) {
+			e.printStackTrace();
+			return DLStatus.INCOMPATIBLE_ZIP;
+		} catch (IOException e) {
+			e.printStackTrace();
+			return DLStatus.INCOMPATIBLE_ZIP;
+		}
+				
+		return super.setModel(zipUri);
+	}
+	
+	/**
+	 * Find the first file within a zip folder that matches a pattern. 
+	 * @param zipUri - uri to the zip file
+	 * @param filePattern - the file pattern to match - the file must contain this string. 
+	 * @return null if no file found and the file pqth if the file is founf
+	 * @throws ZipException
+	 * @throws IOException
+	 */
+	private static String getZipFilePath(URI zipUri, String filePattern) throws ZipException, IOException {
+		return ArchiveModelWorker.getZipFilePath(new File(zipUri),  filePattern);
+	}
+	
+	/**
+	 * Zip test.
+	 * @param args
+	 */
+	public static void main(String[] args) {
+	
+		String fileName = "D:/Dropbox/PAMGuard_dev/Deep_Learning/Gibbons/gibbon_model_shared/gibbon_model.zip";
+		
+		URI zipUri = new  File(fileName).toURI();
+				
+		try {
+			String pbFile = getZipFilePath( zipUri, ".pb");
+			String transformFile = getZipFilePath( zipUri, ".pdtf");
+			
+			System.out.println("pbFile: "  + pbFile + " transformFile: " + transformFile);
+
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+	
+	
+	 
+	
+}
+	
+

src/rawDeepLearningClassifier/dlClassification/archiveModel/SimpleArchiveModel.java

@@ -2,6 +2,12 @@ package rawDeepLearningClassifier.dlClassification.archiveModel;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.jamdev.jdl4pam.ArchiveModel;
 
@@ -14,21 +20,37 @@ import ai.djl.MalformedModelException;
  */
 public class SimpleArchiveModel extends ArchiveModel {
 
+
 	public SimpleArchiveModel(File file) throws MalformedModelException, IOException {
 		super(file);
-		// TODO Auto-generated constructor stub
 	}
 
 	@Override
 	public String getAudioReprRelPath(String zipFolder) {
-		// settings are in parent directory
-		return "audio_repr_pg.json";
+		try {
+			System.out.println("SETTINGS PATH: " + getRelFilePath(zipFolder, ".pdtf"));
+			return  getRelFilePath(zipFolder, ".pdtf");
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+			return null;
+		}
 	}
 
 	@Override
 	public String getModelRelPath(String zipFolder) {
-		// model is in parent directory
-		return "model/saved_model.pb";
+		try {
+			String model = null;
+			model = getRelFilePath(zipFolder, ".pb");
+			if (model==null) model = getRelFilePath(zipFolder, ".py");
+			System.out.println("MODEL PATH: " +model);
+			return model;
+
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+			return null;
+		}
 	}
 
 	@Override
@@ -36,4 +58,28 @@ public class SimpleArchiveModel extends ArchiveModel {
 		return "zip_model";
 	}
 
+	
+	private static String getRelFilePath(String zipFolder, String fileEnd) throws IOException {
+		  // find files matched `png` file extension from folder C:\\test
+		  try (Stream<Path> walk = Files.walk(Paths.get(zipFolder))) {
+		      List<String> result = walk
+		              .filter(p -> !Files.isDirectory(p))   // not a directory
+		              .map(p -> p.toString().toLowerCase()) // convert path to string
+		              .filter(f -> f.endsWith(fileEnd))       // check end with
+		              .collect(Collectors.toList());        // collect all matched to a List
+		      
+		      if (result.size()>0) {
+		      String firstFile = result.get(0); 
+		      
+		      String relative = new File(zipFolder).toURI().relativize(new File(firstFile).toURI()).getPath();
+		      
+		      return relative;
+		      }
+		      else return null;
+		  }
+		  catch (Exception e) {
+			  e.printStackTrace();
+			  return null;
+		  }
+	}
 }

src/rawDeepLearningClassifier/dlClassification/archiveModel/ZipModelClassifier.java

@@ -1,29 +0,0 @@
-package rawDeepLearningClassifier.dlClassification.archiveModel;
-
-import rawDeepLearningClassifier.DLControl;
-
-public class ZipModelClassifier extends ArchiveModelClassifier {
-
-
-	public static String MODEL_NAME = "Zip";
-	
-	/**
-	 * The file extensions
-	 */
-	private String[] fileExtensions = new String[] {"zip"};
-
-
-	public ZipModelClassifier(DLControl dlControl) {
-		super(dlControl);
-	}
-	
-	@Override
-	public String[] getFileExtensions() {
-		return fileExtensions;
-	}
-	
-	@Override
-	public String getName() {
-		return MODEL_NAME;
-	}
-}

src/rawDeepLearningClassifier/dlClassification/dummyClassifier/DummyClassifier.java

@@ -94,9 +94,9 @@ public class DummyClassifier implements DLClassiferModel{
 	}
 
 	@Override
-	public void setModel(URI model) {
+	public DLStatus setModel(URI model) {
 		// TODO Auto-generated method stub
-		
+		return null;
 	}
 
 }

src/rawDeepLearningClassifier/dlClassification/koogu/KooguModelWorker.java

@@ -9,6 +9,12 @@ import org.jamdev.jdl4pam.koogu.KooguModel;
 import ai.djl.MalformedModelException;
 import rawDeepLearningClassifier.dlClassification.archiveModel.ArchiveModelWorker;
 
+/**
+ * The Koogu model worker. 
+ * 
+ * @author Jamie Macaulay
+ *
+ */
 public class KooguModelWorker extends ArchiveModelWorker {
 
 	

src/rawDeepLearningClassifier/dlClassification/orcaSpot/OrcaSpotClassifier.java

@@ -358,9 +358,9 @@ public class OrcaSpotClassifier implements DLClassiferModel, PamSettings {
 	}
 
 	@Override
-	public void setModel(URI model) {
+	public DLStatus setModel(URI model) {
 		// TODO Auto-generated method stub
-		
+		return null;
 	}
 
 }

src/rawDeepLearningClassifier/layoutFX/DLModelSelectPane.java

@@ -375,12 +375,12 @@ public class DLModelSelectPane extends PamBorderPane {
 			
 			try {
 				//we are loading model from a file - anything can happen so put in a try catch. 
-				currentClassifierModel.setModel(file);
-				
-				if (currentClassifierModel.getModelStatus().isError()) {
+				DLStatus status = currentClassifierModel.setModel(file);
+			
+				if (status.isError()) {
 					System.err.println("Model load failed: " + currentClassifierModel.getModelStatus());
 					currentClassifierModel=null;
-					return DLStatus.MODEL_LOAD_FAILED;
+					return status;
 				}
 				
 				return DLStatus.MODEL_LOAD_SUCCESS;