nordic-dev.net/PAMGuard
nordic-dev.net/PAMGuard
2
0
Fork 0
mirror of https://github.com/PAMGuard/PAMGuard.git synced 2024-11-26 00:52:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Deep learning memory leak fix

Browse Source 
Fixed a memory leak whereby reloading the model every file (even if the model is closed) caused a memory leak.
This commit is contained in:
Jamie Mac 2022-11-21 15:56:43 +00:00
parent 503252015a
commit b3184fcafe
5 changed files with 47 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/rawDeepLearningClassifier/dlClassification/DLClassifyProcess.java
View File

@ -36,6 +36,7 @@ import rawDeepLearningClassifier.segmenter.SegmenterProcess.GroupedRawData;
*/
public class DLClassifyProcess extends PamInstantProcess {
/**
* Holds all model results but no other information
*/
@ -538,7 +539,9 @@ public class DLClassifyProcess extends PamInstantProcess {
@Override
public void pamStop() {
runModel(); //make sure to run the last data in the buffer.
this.dlControl.getDLModel().closeModel();
//21/11/2022 - it seems like this causes a memory leak when models are reopened and closed every file...
//this.dlControl.getDLModel().closeModel();
}
/**

6
src/rawDeepLearningClassifier/dlClassification/genericModel/GenericDLClassifier.java
View File

@ -168,7 +168,11 @@ public class GenericDLClassifier implements DLClassiferModel, PamSettings {
@Override
public void closeModel() {
// TODO Auto-generated method stub
//very important to prevent memory leak for long term processing.
if (genericModelWorker.getModel()!=null && genericModelWorker.getModel().getModel()!=null) {
//System.out.println("CLOSE GENERNIC MODEL");
genericModelWorker.getModel().getModel().close();
}
}

16
src/rawDeepLearningClassifier/dlClassification/genericModel/GenericModelWorker.java
View File

@ -1,5 +1,7 @@
package rawDeepLearningClassifier.dlClassification.genericModel;
import java.nio.file.Paths;
import org.apache.commons.io.FilenameUtils;
import org.jamdev.jdl4pam.genericmodel.GenericModel;
import org.jamdev.jdl4pam.transforms.DLTransform;
@ -67,11 +69,25 @@ public class GenericModelWorker extends DLModelWorker<GenericPrediction> {
// if (DLControl.PLUGIN_BUILD) {
// PluginClassloader newCL = PamModel.getPamModel().getClassLoader();
// Thread.currentThread().setContextClassLoader(newCL);
// }
// if (genericModel!=null) {
// System.out.println(Paths.get(genericModel.getModel().getName()));
// System.out.println(Paths.get(genericParams.modelPath));
// System.out.println(Paths.get(genericModel.getModel().getName()).equals(Paths.get(genericParams.modelPath)));
// }
//first open the model and get the correct parameters.
//21/11/2022 - Added a null and filename check here to stop the mdoel reloading everytime PAMGuard hits a new file or
//is stopped or started - this was causing a memory leak.
if (genericModel==null || !Paths.get(genericModel.getModel().getName()).equals(Paths.get(genericParams.modelPath))) {
//System.out.println(Paths.get(genericParams.modelPath));
genericModel = new PamGenericModel(genericParams.modelPath);
//System.out.println("LOAD A NEW MODEL: ");
//System.out.println(genericModel.getModel().getModelPath().getFileName());
}
//is this a waveform or a spectrogram model?
DLTransform transform = genericParams.dlTransfroms.get(genericParams.dlTransfroms.size()-1);
if (transform instanceof FreqTransform) {

6
src/rawDeepLearningClassifier/dlClassification/genericModel/PamGenericModel.java
View File

@ -65,6 +65,8 @@ public class PamGenericModel {
public PamGenericModel(String modelPath) throws MalformedModelException, IOException{
//System.out.println("NEW GENERIC MODEL:");
File file = new File(modelPath);
//String modelPath = "/Users/au671271/Google Drive/Aarhus_research/PAMGuard_bats_2020/deep_learning/BAT/models/bats_denmark/BAT_4ms_256ft_8hop_128_NOISEAUG_40000_100000_-100_0_256000_JAMIE.pk";
@ -95,7 +97,6 @@ public class PamGenericModel {
if (model == null) {
System.err.println("Generic Model: Could not load model: " + modelPath);
}
else {
if (model!=null && model.describeInput()!=null) {
System.out.println("Generic Model: Input: " + model.describeInput().toString());
@ -117,7 +118,6 @@ public class PamGenericModel {
//predictor for the model if using
wavePredictor = model.newPredictor(waveTranslator);
}
}
@ -201,8 +201,6 @@ public class PamGenericModel {
}
public Model getModel() {
return model;
}

15
src/rawDeepLearningClassifier/dlClassification/ketos/KetosWorker.java
View File

@ -1,6 +1,7 @@
package rawDeepLearningClassifier.dlClassification.ketos;
import java.io.File;
import java.nio.file.Paths;
import java.util.ArrayList;
import org.jamdev.jdl4pam.ketos.KetosModel;
@ -15,6 +16,7 @@ import PamModel.PamModel.PluginClassloader;
import rawDeepLearningClassifier.DLControl;
import rawDeepLearningClassifier.dlClassification.animalSpot.StandardModelParams;
import rawDeepLearningClassifier.dlClassification.genericModel.DLModelWorker;
import rawDeepLearningClassifier.dlClassification.genericModel.PamGenericModel;
/**
*
@ -32,6 +34,11 @@ public class KetosWorker extends DLModelWorker<KetosResult> {
*/
private KetosModel ketosModel;
/**
* Thelast loaded model path.,
*/
private String currentPath;
/**
* SoundSpotWorker constructor.
@ -56,7 +63,15 @@ public class KetosWorker extends DLModelWorker<KetosResult> {
Thread.currentThread().setContextClassLoader(newCL);
}
//first open the model and get the correct parameters.
//21/11/2022 - Added a null and filename check here to stop the mdoel reloading everytime PAMGuard hits a new file or
//is stopped or started - this was causing a memory leak.
if (ketosModel==null || currentPath ==null || !Paths.get(currentPath).equals(Paths.get(ketosDLParams.modelPath))) {
//System.out.println(Paths.get(genericParams.modelPath));
this.currentPath = ketosDLParams.modelPath;
ketosModel = new KetosModel(new File(ketosDLParams.modelPath));
//System.out.println("LOAD A NEW MODEL: ");
//System.out.println(genericModel.getModel().getModelPath().getFileName());
}
}
catch (Exception e) {
e.printStackTrace();