mirror of
https://github.com/PAMGuard/PAMGuard.git
synced 2024-11-25 08:32:32 +00:00
Rewrite import thread code because it's too clunky and slow.
This commit is contained in:
Jamie Mac
parent
66fed778a3
commit
934000fd9c
@ -319,7 +319,7 @@ public class CPODControl2 extends PamControlledUnit implements PamSettings {
|
||||
* @param files - the files to import.
|
||||
* @return a list of import Tasks.
|
||||
*/
|
||||
public List<Task<Integer>> importPODData(List<File> files) {
|
||||
public Task<Integer> importPODData(List<File> files) {
|
||||
return cpodImporter.importCPODData(files);
|
||||
}
|
||||
|
||||
|
||||
45
src/cpod/CPODFile.java
Normal file
45
src/cpod/CPODFile.java
Normal file
@ -0,0 +1,45 @@
|
||||
package cpod;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import cpod.CPODUtils.CPODFileType;
|
||||
|
||||
/**
|
||||
* A CP1 along with it's CP3 file
|
||||
* @author Jamie Macaulay
|
||||
*
|
||||
*/
|
||||
public class CPODFile {
|
||||
|
||||
/**
|
||||
* A CP1 or FP1 file
|
||||
*/
|
||||
public File cp1File = null;
|
||||
|
||||
/**
|
||||
* A CP3 or CP3 file
|
||||
*/
|
||||
public File cp3File = null;
|
||||
|
||||
/**
|
||||
* Get the name of the file
|
||||
* @return the name of the file;
|
||||
*/
|
||||
public String getName() {
|
||||
String fileName = "";
|
||||
if (cp1File!=null) fileName+=cp1File.getName() + " ";
|
||||
if (cp3File!=null) fileName+=cp3File.getName() + " ";
|
||||
return fileName;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether the current CPOD file is a CPOD or FPOD
|
||||
* @return true if FPOD.
|
||||
*/
|
||||
public boolean isFPOD() {
|
||||
if (cp1File !=null) return CPODUtils.getFileType(cp1File).equals(CPODFileType.FP1);
|
||||
if (cp3File !=null) return CPODUtils.getFileType(cp3File).equals(CPODFileType.FP3);
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,6 +1,7 @@
|
||||
package cpod;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.Collections;
|
||||
@ -39,22 +40,7 @@ public class CPODImporter {
|
||||
|
||||
public static final int MAX_SAVE = 2000000;
|
||||
|
||||
|
||||
private long fileStart;
|
||||
|
||||
private CPODControl2 cpodControl;
|
||||
float[] tempDataGramData;
|
||||
|
||||
// /**
|
||||
// * Flag for a CP1 file.
|
||||
// */
|
||||
// public static final int FILE_CP1 = 1;
|
||||
//
|
||||
// /**
|
||||
// * Flag for a CP3 file.
|
||||
// */
|
||||
// public static final int FILE_CP3 = 3;
|
||||
|
||||
|
||||
/**
|
||||
* Handles the queue for importing files tasks
|
||||
@ -66,28 +52,11 @@ public class CPODImporter {
|
||||
});
|
||||
|
||||
|
||||
|
||||
|
||||
public CPODImporter(CPODControl2 cpodControl) {
|
||||
this.cpodControl = cpodControl;
|
||||
// if (fileType != cpFileType) {
|
||||
// System.err.println("CPOD Mismatched file type " + cpFile.getAbsolutePath());
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// if (cpFileType == FILE_CP1) {
|
||||
// dataBlock = cpodControl.getCP1DataBlock();
|
||||
// }
|
||||
// else {
|
||||
// dataBlock = cpodControl.getCP3DataBlock();
|
||||
//
|
||||
// }
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Process a CPOD click.
|
||||
* @param FPODData - an FPOD data object
|
||||
@ -105,8 +74,6 @@ public class CPODImporter {
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Process the click.
|
||||
* @param FPODData - an FPOD data object
|
||||
@ -126,34 +93,10 @@ public class CPODImporter {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return the fileStart
|
||||
*/
|
||||
public long getFileStart() {
|
||||
return fileStart;
|
||||
}
|
||||
|
||||
// /**
|
||||
// * Get the data
|
||||
// * @param type
|
||||
// * @return
|
||||
// */
|
||||
// public CPODClickDataBlock getDataBlock(CPODFileType type) {
|
||||
// switch (type) {
|
||||
// case CP1:
|
||||
// case FP1:
|
||||
// return this.cpodControl.getCP1DataBlock();
|
||||
// case CP3:
|
||||
// case FP3:
|
||||
// return this.cpodControl.getCP3DataBlock();
|
||||
// }
|
||||
// return null;
|
||||
// }
|
||||
|
||||
/**
|
||||
* Run the import task.
|
||||
*/
|
||||
public void runImportTask(ArrayList<File> files, CPODClickDataBlock clickDataBlock, CPODClickTrainDataBlock clickTrainDataBlock) {
|
||||
public void runImportTask(ArrayList<CPODFile> files, CPODClickDataBlock clickDataBlock, CPODClickTrainDataBlock clickTrainDataBlock) {
|
||||
Task<Integer> cpodTask = new CPODImportTask(files, clickDataBlock, clickTrainDataBlock);
|
||||
|
||||
Thread th = new Thread(cpodTask);
|
||||
@ -163,61 +106,102 @@ public class CPODImporter {
|
||||
th.start();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Import the CPOD data from a certain file type.
|
||||
* @param files to import - a list of CPOD compatible files (can be a mix but only the files corresponding to type will be processed)
|
||||
* @param type - the type flag of the file e.g. CPODFileType.CP1
|
||||
* @return the CPOD import task.
|
||||
*/
|
||||
public Task<Integer> importCPODDataTask(List<File> files, CPODFileType type) {
|
||||
public Task<Integer> importCPODDataTask(List<CPODFile> files) {
|
||||
|
||||
List<File> cpXFIles = new ArrayList<File>();
|
||||
|
||||
for (int i=0; i<files.size(); i++) {
|
||||
String ext = FilenameUtils.getExtension(files.get(i).getAbsolutePath());
|
||||
if (ext.equals(type.getText())) {
|
||||
cpXFIles.add(files.get(i));
|
||||
}
|
||||
}
|
||||
|
||||
CPODImportTask cpodTask = new CPODImportTask(cpXFIles, cpodControl.getCP1DataBlock(), cpodControl.getClickTrainDataBlock());
|
||||
CPODImportTask cpodTask = new CPODImportTask(files, cpodControl.getCP1DataBlock(), cpodControl.getClickTrainDataBlock());
|
||||
|
||||
return cpodTask;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Import the CPOD data from a list of CPOD files.
|
||||
* @param a list of CPOD compatible files (can be a mix)
|
||||
* @return a list of tasks whihc imports each file type.
|
||||
* Import CPOD and/or FPOD data from a list of files. The files can be CP1/FP1 or CP3/FP3 or a mix of both.
|
||||
* @param files - a list of CPOD or FPOD files
|
||||
* @return the task to import the list.
|
||||
*/
|
||||
public List<Task<Integer>> importCPODData(List<File> files) {
|
||||
|
||||
List<Task<Integer>> tasks = new ArrayList<Task<Integer>>();
|
||||
|
||||
for (int i=0; i<CPODFileType.values().length; i++) {
|
||||
Task<Integer> cp1Task = importCPODDataTask(files, CPODFileType.values()[i]);
|
||||
tasks.add(cp1Task);
|
||||
public Task<Integer> importCPODData(List<File> files) {
|
||||
// Need to create a list of cp1 and cp3 files.
|
||||
ArrayList<CPODFile> cpodFiles = new ArrayList<CPODFile>();
|
||||
|
||||
CPODFile cpodFile;
|
||||
while (files.size()>0) {
|
||||
cpodFile = new CPODFile();
|
||||
|
||||
CPODFileType type = CPODUtils.getFileType(files.get(0));
|
||||
|
||||
File currentFile = files.get(0);
|
||||
switch (type) {
|
||||
case CP1:
|
||||
case FP1:
|
||||
cpodFile.cp1File= files.get(0);
|
||||
break;
|
||||
case CP3:
|
||||
case FP3:
|
||||
cpodFile.cp3File= files.get(0);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
|
||||
}
|
||||
files.remove(0);
|
||||
|
||||
//now search for the corresponding CP3 or CP1 file
|
||||
for (int i=0; i<files.size(); i++) {
|
||||
if (isFilePair(currentFile, files.get(i))){
|
||||
switch (type) {
|
||||
case CP1:
|
||||
case FP1:
|
||||
cpodFile.cp3File= files.get(i);
|
||||
break;
|
||||
case CP3:
|
||||
case FP3:
|
||||
cpodFile.cp1File= files.get(i);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
|
||||
}
|
||||
break;//break out of the for loop
|
||||
}
|
||||
}
|
||||
|
||||
cpodFiles.add(cpodFile);
|
||||
|
||||
}
|
||||
|
||||
tasks.get(tasks.size()-1).setOnSucceeded((workerState)->{
|
||||
PamController.getInstance().updateDataMap();
|
||||
});
|
||||
|
||||
//TODO what if a task is cancelled...
|
||||
return tasks;
|
||||
|
||||
return new CPODImportTask(cpodFiles, this.cpodControl.getCP1DataBlock(), this.cpodControl.getClickTrainDataBlock());
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether the two files are a CP1, CP3 pair or FP1 FP3 pair.
|
||||
* @param cpxfile1
|
||||
* @param cpxFile2
|
||||
* @return true if the files are a pair.
|
||||
*/
|
||||
private boolean isFilePair(File cpxfile1, File cpxFile2) {
|
||||
|
||||
String filePattern = cpxfile1.getName().substring(0, cpxfile1.getName().length()-1);
|
||||
|
||||
if (cpxFile2.getName().contains(filePattern) && !cpxfile1.getName().equals(cpxFile2.getName())) {
|
||||
return true;
|
||||
}
|
||||
else return false;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Run the tasks
|
||||
* @param tasks - the tasks.
|
||||
* @param task - the tasks.
|
||||
*/
|
||||
public void runTasks(List<Task<Integer>> tasks) {
|
||||
|
||||
for (int i=0; i<CPODFileType.values().length; i++) {
|
||||
this.exec.execute(tasks.get(i));
|
||||
}
|
||||
public void runTasks(Task<Integer> task) {
|
||||
this.exec.execute(task);
|
||||
}
|
||||
|
||||
|
||||
@ -232,7 +216,7 @@ public class CPODImporter {
|
||||
/**
|
||||
* List of files, either CP1 or CP3
|
||||
*/
|
||||
private List<File> cpxFile;
|
||||
private List<CPODFile> cpxFile;
|
||||
|
||||
/**
|
||||
* Reference to the binary store.
|
||||
@ -261,18 +245,17 @@ public class CPODImporter {
|
||||
* @param cpxfiles - a list of CP1 or CP3 files.
|
||||
* @param cpodDataBlock - the CPOD data block.
|
||||
*/
|
||||
public CPODImportTask(List<File> cpxfiles, CPODClickDataBlock cpodDataBlock, CPODClickTrainDataBlock clickTrainDataBlock) {
|
||||
public CPODImportTask(List<CPODFile> cpxfiles, CPODClickDataBlock cpodDataBlock, CPODClickTrainDataBlock clickTrainDataBlock) {
|
||||
this.cpxFile = cpxfiles;
|
||||
this.cpodDataBlock=cpodDataBlock;
|
||||
this.clickTrainDataBlock=clickTrainDataBlock;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
protected Integer call() throws Exception {
|
||||
try {
|
||||
|
||||
|
||||
BinaryDataSource binarySource = cpodDataBlock.getBinaryDataSource();
|
||||
binaryStore = (BinaryStore) PamController.getInstance().findControlledUnit(BinaryStore.defUnitType);
|
||||
if (binaryStore == null) {
|
||||
@ -293,29 +276,27 @@ public class CPODImporter {
|
||||
if (this.isCancelled()) return -1;
|
||||
|
||||
final int ii = i;
|
||||
|
||||
|
||||
System.out.println(("Importing CPOD file: " + (ii+1)) + " " +cpxFile.get(i));
|
||||
this.updateMessage(("Importing CPOD file: " + (ii+1)) + " " +cpxFile.get(i).getName());
|
||||
|
||||
int nClicks = 0;
|
||||
int totalClicks = Integer.MAX_VALUE;
|
||||
int importedClicks = -1;
|
||||
|
||||
this.updateProgress(-1, 1);
|
||||
|
||||
long[] fileStartEnd = getFileStartEnd(cpxFile.get(i));
|
||||
//get the start and end of the file.
|
||||
long[] fileStartEnd = getFileStartEnd(cpxFile.get(i).cp1File !=null ? cpxFile.get(i).cp1File : cpxFile.get(i).cp3File);
|
||||
|
||||
while (importedClicks>0 || importedClicks==-1) {
|
||||
|
||||
System.out.println(("Importing file " + i + " of " + cpxFile.size() + " from detection " + nClicks));
|
||||
|
||||
CPODFileType fileType = CPODUtils.getFileType(cpxFile.get(i));
|
||||
|
||||
System.out.println(("Importing from " + fileType + " " + i + " of " + cpxFile.size() + " from detection " + nClicks));
|
||||
this.updateMessage(("Importing from " + fileType + " " + i + " of " + cpxFile.size() + " from detection " + nClicks));
|
||||
this.updateMessage(("Importing file " + i + " of " + cpxFile.size() + " from detection " + nClicks));
|
||||
|
||||
//import the CPOD or FPOD data
|
||||
this.updateProgress(-1, 1);
|
||||
importedClicks = importFile(cpxFile.get(i), cpodDataBlock, clickTrainDataBlock, nClicks, MAX_SAVE);
|
||||
importedClicks = importCPODFile(cpxFile.get(i), cpodDataBlock, clickTrainDataBlock, nClicks, MAX_SAVE);
|
||||
this.updateProgress(-1, 1);
|
||||
|
||||
System.out.println("Number of CPOD data units in the data block: " + cpodDataBlock.getUnitsCount() + " progress: " + (i+1) + " " + cpxFile.size() );
|
||||
@ -348,60 +329,20 @@ public class CPODImporter {
|
||||
double progress=(((double) click.getTimeMilliseconds()-fileStartEnd[0]))/(fileStartEnd[1]-fileStartEnd[0]);
|
||||
this.updateProgress(i+(progress), cpxFile.size());
|
||||
|
||||
|
||||
if (day>-1) {
|
||||
|
||||
//save left over CP1 data
|
||||
if (cpodTrainList!=null) {
|
||||
//all duplicate clicks will have been removed. Save the data now
|
||||
for (int j=0; j<cpodTrainList.size(); j++) {
|
||||
data = cpodDataBlock.getBinaryDataSource().getPackedData(cpodTrainList.get(j));
|
||||
this.binaryStream.storeData(data.getObjectType(), click.getBasicData(), data);
|
||||
}
|
||||
count = 0;
|
||||
}
|
||||
|
||||
//close current file
|
||||
binaryStream.writeModuleFooter();
|
||||
binaryStream.writeFooter(click.getTimeMilliseconds(), System.currentTimeMillis(), BinaryFooter.END_UNKNOWN);
|
||||
binaryStream.closeFile();
|
||||
binaryStream.createIndexFile();
|
||||
|
||||
}
|
||||
|
||||
System.out.println("Open new binary file: " + PamCalendar.formatDBDateTime(click.getTimeMilliseconds()));
|
||||
|
||||
//send an update message
|
||||
final String timeMillisStr = PamCalendar.formatDBDateTime(click.getTimeMilliseconds());
|
||||
Platform.runLater(()->{
|
||||
this.updateMessage(("Saving file: " + timeMillisStr));
|
||||
});
|
||||
|
||||
//get a list of the currently loaded click data if this is CP3 data
|
||||
if (fileType.equals(CPODFileType.CP3) || fileType.equals(CPODFileType.FP3)) {
|
||||
|
||||
cpodDataBlock.clearAll();
|
||||
|
||||
System.err.println("There are already data units from CP1 or FP1 files " + cpodDataBlock.getUnitsCount());
|
||||
|
||||
//load the view from the data block for that day.
|
||||
long millisDayStart = roundToDay(click.getTimeMilliseconds());
|
||||
cpodDataBlock.loadViewerData(millisDayStart, millisDayStart+24*60*60*1000L-1, null);
|
||||
|
||||
System.out.println("Load viewer data from: " + PamCalendar.formatDBDateTime(millisDayStart) +
|
||||
" to " +PamCalendar.formatDBDateTime(millisDayStart+24*60*60*1000L-1));
|
||||
|
||||
//now the tricky bit - if we are loading viewer files we want to save the data from the CP3 on top of the
|
||||
//CP1 file but we DO NOT want to add in duplicate clicks.
|
||||
if (cpodDataBlock.getUnitsCount()>0) {
|
||||
System.out.println("There are already data units from CP1 or FP1 files " + cpodDataBlock.getUnitsCount());
|
||||
cpodTrainList=null;
|
||||
cpodTrainList = cpodDataBlock.getDataCopy();
|
||||
Collections.sort(cpodTrainList, new CPODClickOmparator());
|
||||
cpodDataBlock.clearAll(); //just incase
|
||||
}
|
||||
|
||||
}
|
||||
this.updateMessage(("Saving file: " + timeMillisStr));
|
||||
|
||||
//write the module head
|
||||
binaryStream.openOutputFiles(click.getTimeMilliseconds());
|
||||
@ -411,20 +352,6 @@ public class CPODImporter {
|
||||
day=dayYear;
|
||||
}
|
||||
|
||||
if (cpodTrainList!=null && cpodTrainList.size()>0) {
|
||||
// index of the target
|
||||
int index = Collections.binarySearch(
|
||||
cpodTrainList, click, new CPODClickOmparator());
|
||||
|
||||
if (index>-1) {
|
||||
//the click in the cp3 file is present in the CP1 file. Save this click as it has a super detection. DO NOT save the
|
||||
//CP1 click;
|
||||
cpodTrainList.remove(index);
|
||||
count++;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
data = cpodDataBlock.getBinaryDataSource().getPackedData(click);
|
||||
this.binaryStream.storeData(data.getObjectType(), click.getBasicData(), data);
|
||||
}
|
||||
@ -432,7 +359,7 @@ public class CPODImporter {
|
||||
cpodDataBlock.clearAll();
|
||||
|
||||
//update number of clicks.
|
||||
nClicks=nClicks+MAX_SAVE+1;
|
||||
nClicks=nClicks+importedClicks+1;
|
||||
}
|
||||
|
||||
}
|
||||
@ -464,7 +391,7 @@ public class CPODImporter {
|
||||
calendar.set(Calendar.MILLISECOND, 0);
|
||||
return calendar.getTimeInMillis();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Import the CPOD file.
|
||||
* @param cpFile2 - the cp1 file
|
||||
@ -473,16 +400,55 @@ public class CPODImporter {
|
||||
* @param maxNum - the maximum number to import
|
||||
* @return the total number of clicks in the file.
|
||||
*/
|
||||
private int importCPODFile(File cpFile2, CPODClickDataBlock dataBlock, CPODClickTrainDataBlock clickTrainDataBlock, int from, int maxNum) {
|
||||
ArrayList<CPODClick> cpodData = null;
|
||||
private int importCPODFile(CPODFile cpFile, CPODClickDataBlock dataBlock, CPODClickTrainDataBlock clickTrainDataBlock, int from, int maxNum) {
|
||||
ArrayList<CPODClick> cpodCP1Data = null;
|
||||
ArrayList<CPODClick> cpodCP3Data = null;
|
||||
|
||||
try {
|
||||
cpodData = CPODReader.importCPODFile(cpFile2, from, maxNum);
|
||||
|
||||
if (cpFile.isFPOD()) {
|
||||
//load a chunk of FP1 data
|
||||
cpodCP1Data = FPODReader.importFPODFile(cpFile.cp1File, from, maxNum);
|
||||
//load all FP3 data
|
||||
cpodCP3Data = FPODReader.importFPODFile(cpFile.cp3File, 0, Integer.MAX_VALUE);
|
||||
}
|
||||
else {
|
||||
//load a chunk of CP1 data
|
||||
cpodCP1Data = CPODReader.importCPODFile(cpFile.cp1File, from, maxNum);
|
||||
//load all CP3 data
|
||||
cpodCP3Data = CPODReader.importCPODFile(cpFile.cp3File, 0, Integer.MAX_VALUE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
return 0;
|
||||
}
|
||||
|
||||
//create an arraylist
|
||||
ArrayList<CPODClick> cpodData = new ArrayList<CPODClick>();
|
||||
|
||||
|
||||
//now we get rid if duplicate clicks
|
||||
if (cpodCP1Data != null && cpodCP3Data != null) {
|
||||
CPODClickOmparator cpodComparator = new CPODClickOmparator();
|
||||
Collections.sort(cpodCP3Data, cpodComparator);;
|
||||
//here we need to replace the CP1 detection in the CP1 file with the CP3 detections in the CP3 file
|
||||
for (int j=0; j<cpodCP1Data.size(); j++) {
|
||||
int index = Collections.binarySearch(cpodCP3Data, cpodCP1Data.get(j),cpodComparator);
|
||||
//replace
|
||||
if (index>=0) {
|
||||
cpodCP1Data.set(j, cpodCP3Data.get(index));
|
||||
}
|
||||
}
|
||||
cpodData= cpodCP1Data; //cp1 data now contains CP3 clicks.
|
||||
}
|
||||
else if (cpodCP1Data != null) cpodData= cpodCP1Data; //only CP1 data imported
|
||||
else if (cpodCP3Data != null) cpodData= cpodCP3Data; //only CP3 data imported
|
||||
|
||||
|
||||
|
||||
HashMap<Integer, CPODClickTrainDataUnit> cpodClickTrains = new HashMap<Integer, CPODClickTrainDataUnit>();
|
||||
|
||||
// fileStart + nMinutes * 60000L;
|
||||
@ -494,19 +460,19 @@ public class CPODImporter {
|
||||
dataBlock.addPamData(cpodClick);
|
||||
if (cpodData.get(i).getClassification()!=null) {
|
||||
CPODClickTrainDataUnit clickTrain = cpodClickTrains.get(cpodData.get(i).getClassification().clicktrainID);
|
||||
|
||||
|
||||
//add the click train to the hash map
|
||||
if (clickTrain==null) {
|
||||
clickTrain= new CPODClickTrainDataUnit(cpodClick.getTimeMilliseconds(), null, cpodData.get(i).getClassification());
|
||||
cpodClickTrains.put(cpodData.get(i).getClassification().clicktrainID, clickTrain);
|
||||
}
|
||||
|
||||
|
||||
//add the cpos click as a sub detection
|
||||
clickTrain.addSubDetection(cpodClick);
|
||||
}
|
||||
nClicks++;
|
||||
}
|
||||
|
||||
|
||||
//add all the click trains with sub detections ot the datablock.
|
||||
int count =0;
|
||||
for (Integer key: cpodClickTrains.keySet()) {
|
||||
@ -516,7 +482,7 @@ public class CPODImporter {
|
||||
this.updateProgress(count, cpodClickTrains.keySet().size());
|
||||
// System.out.println("Add click train data to datablock: " + count + " " + cpodClickTrains.keySet().size());
|
||||
}
|
||||
|
||||
|
||||
clickTrainDataBlock.addPamData(cpodClickTrains.get(key));
|
||||
count++;
|
||||
}
|
||||
@ -527,140 +493,45 @@ public class CPODImporter {
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Import a FPOD file.
|
||||
* @param cpFile - the FP1 or FP3 file.
|
||||
* @param from - the click index to save from. e.g. 100 means that only click 100 + in the file is saved
|
||||
* @param maxNum - the maximum number to import
|
||||
* @return the total number of clicks in the file.
|
||||
*/
|
||||
protected int importFPODFile(File cpFile, CPODClickDataBlock dataBlock, CPODClickTrainDataBlock clickTrainDataBlock, int from, int maxNum) {
|
||||
|
||||
/**
|
||||
* Note that this function is very simialr to the CPOD ijport v- we could abstract it but, ther eis likely to be a few changes to these fucntions a
|
||||
* as FPOD data is further developed and so making this enat may be far more trouble than it worth.
|
||||
*/
|
||||
|
||||
ArrayList<FPODdata> fpodData = new ArrayList<FPODdata>();
|
||||
|
||||
try {
|
||||
FPODReader.importFile(cpFile, fpodData, from, maxNum);
|
||||
|
||||
// fileStart + nMinutes * 60000L;
|
||||
|
||||
HashMap<Integer, CPODClickTrainDataUnit> cpodClickTrains = new HashMap<Integer, CPODClickTrainDataUnit>();
|
||||
|
||||
int nClicks = 0;
|
||||
for (int i=0; i<fpodData.size(); i++) {
|
||||
if (i%40000 ==0) {
|
||||
System.out.println("Create a new CPOD click: " + i + " of " + fpodData.size() + PamCalendar.formatDateTime(fpodData.get(i).getTimeMillis()));
|
||||
}
|
||||
CPODClick cpodClick = processFPODClick(fpodData.get(i));
|
||||
dataBlock.addPamData(cpodClick);
|
||||
|
||||
if (fpodData.get(i).getClassification()!=null) {
|
||||
CPODClickTrainDataUnit clickTrain = cpodClickTrains.get(fpodData.get(i).getClassification().clicktrainID);
|
||||
if (clickTrain==null) {
|
||||
clickTrain= new CPODClickTrainDataUnit(cpodClick.getTimeMilliseconds(), null, fpodData.get(i).getClassification());
|
||||
cpodClickTrains.put(fpodData.get(i).getClassification().clicktrainID, clickTrain);
|
||||
}
|
||||
clickTrain.addSubDetection(cpodClick);
|
||||
}
|
||||
nClicks++;
|
||||
}
|
||||
|
||||
System.out.println("Add click train to datablock: ");
|
||||
|
||||
//add all the click trains with sub detections ot the datablock.
|
||||
int count =0;
|
||||
for (Integer key: cpodClickTrains.keySet()) {
|
||||
|
||||
if (count%100 ==0) {
|
||||
this.updateMessage(("Add click train data to datablock: " + count + " of " + cpodClickTrains.keySet().size()));
|
||||
this.updateProgress(count, cpodClickTrains.keySet().size());
|
||||
|
||||
// System.out.println("Add click train data to datablock: " + count + " " + cpodClickTrains.keySet().size());
|
||||
}
|
||||
|
||||
clickTrainDataBlock.addPamData(cpodClickTrains.get(key));
|
||||
count++;
|
||||
}
|
||||
|
||||
System.out.println("Finished adding click train to datablock: ");
|
||||
|
||||
|
||||
fpodData=null; //trigger garbage collector if needed
|
||||
|
||||
return nClicks;
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Import a CPOD or FPOD file.
|
||||
* @param cpFile - the CP1/FP1 or CP3/FP3 file.
|
||||
* @param from - the click index to save from. e.g. 100 means that only click 100 + in the file is saved
|
||||
* @param maxNum - the maximum number to import
|
||||
* @return the total number of clicks in the file.
|
||||
*/
|
||||
protected int importFile(File cpFile, CPODClickDataBlock dataBlock, CPODClickTrainDataBlock clickTrainDataBlock, int from, int maxNum) {
|
||||
CPODFileType fileType = CPODUtils.getFileType( cpFile);
|
||||
|
||||
switch (fileType) {
|
||||
case CP1:
|
||||
case CP3:
|
||||
return importCPODFile(cpFile, dataBlock, clickTrainDataBlock, from, maxNum);
|
||||
case FP1:
|
||||
case FP3:
|
||||
return importFPODFile(cpFile, dataBlock, clickTrainDataBlock, from, maxNum);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the start and end of a file from the header data. This opens and closes the file.
|
||||
* @param file -a CP1, CP3 FP1 or FP3 file;
|
||||
* @return the start and end time in Java millis in a long array (first element start, last element end)
|
||||
*/
|
||||
private long[] getFileStartEnd(File file) {
|
||||
CPODFileType fileType = CPODUtils.getFileType(file);
|
||||
long[] timelims = null;
|
||||
switch (fileType){
|
||||
case CP1:
|
||||
case CP3:
|
||||
CPODHeader headerC =CPODReader.readHeader(file);
|
||||
timelims=new long[] {headerC.fileStart, headerC.fileEnd};
|
||||
break;
|
||||
case FP1:
|
||||
case FP3:
|
||||
FPODHeader header =FPODReader.readHeader(file);
|
||||
if (header==null) return null;
|
||||
timelims=new long[] {CPODUtils.podTimeToMillis(header.FirstLoggedMin), CPODUtils.podTimeToMillis(header.LastLoggedMin)};
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return timelims;
|
||||
}
|
||||
|
||||
/**
|
||||
* Import a file.
|
||||
* @param cpFile - the CP1 file.
|
||||
* @return the number of clicks saved to the datablock
|
||||
*/
|
||||
protected int importFile(File cpFile, CPODClickDataBlock dataBlock, CPODClickTrainDataBlock clickTrainDataBlock) {
|
||||
return importFile( cpFile, dataBlock, clickTrainDataBlock, -1, Integer.MAX_VALUE);
|
||||
protected int importCPODFile(CPODFile cpFile, CPODClickDataBlock dataBlock, CPODClickTrainDataBlock clickTrainDataBlock) {
|
||||
return importCPODFile( cpFile, dataBlock, clickTrainDataBlock, -1, Integer.MAX_VALUE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the start and end of a file from the header data. This opens and closes the file.
|
||||
* @param file -a CP1, CP3 FP1 or FP3 file;
|
||||
* @return the start and end time in Java millis in a long array (first element start, last element end)
|
||||
*/
|
||||
private long[] getFileStartEnd(File file) {
|
||||
CPODFileType fileType = CPODUtils.getFileType(file);
|
||||
long[] timelims = null;
|
||||
switch (fileType){
|
||||
case CP1:
|
||||
case CP3:
|
||||
CPODHeader headerC =CPODReader.readHeader(file);
|
||||
timelims=new long[] {headerC.fileStart, headerC.fileEnd};
|
||||
break;
|
||||
case FP1:
|
||||
case FP3:
|
||||
FPODHeader header =FPODReader.readHeader(file);
|
||||
if (header==null) return null;
|
||||
timelims=new long[] {CPODUtils.podTimeToMillis(header.FirstLoggedMin), CPODUtils.podTimeToMillis(header.LastLoggedMin)};
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return timelims;
|
||||
}
|
||||
|
||||
// Comparator to sort a list
|
||||
class CPODClickOmparator implements Comparator<CPODClick> {
|
||||
@Override public int compare(CPODClick s1, CPODClick s2)
|
||||
@ -679,7 +550,8 @@ public class CPODImporter {
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -129,7 +129,9 @@ public class CPODReader {
|
||||
* @param maxNum - the maximum number to import
|
||||
* @return the total number of clicks in the file.
|
||||
*/
|
||||
protected static ArrayList<CPODClick> importCPODFile(File cpFile, int from, int maxNum) {
|
||||
protected static ArrayList<CPODClick> importCPODFile(File cpFile, int from, int maxNum) {
|
||||
|
||||
if (cpFile==null) return null;
|
||||
|
||||
//holds a map of the click train detections.
|
||||
HashMap<Integer, CPODClassification> clickTrains = new HashMap<Integer, CPODClassification>();
|
||||
|
||||
@ -124,4 +124,5 @@ public class CPODUtils {
|
||||
return (short) (data>>firstBit);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -11,6 +11,7 @@ import java.util.HashMap;
|
||||
|
||||
import PamUtils.PamArrayUtils;
|
||||
import PamUtils.PamCalendar;
|
||||
import cpod.FPODReader.FPODdata;
|
||||
|
||||
|
||||
/**
|
||||
@ -99,6 +100,35 @@ public class FPODReader {
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Import FPOD data and return a structure of CPOD clicks.
|
||||
* @param cpFile - the FP1 file.
|
||||
* @param from - the click index to save from. e.g. 100 means that only click 100 + in the file is saved
|
||||
* @param maxNum - the maximum number of data units to import.
|
||||
* @return an array of CPOD clicks.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static ArrayList<CPODClick> importFPODFile(File cpFile, int from, int maxNum ) throws IOException {
|
||||
if (cpFile==null) return null;
|
||||
ArrayList<FPODdata> fpodData = new ArrayList<FPODdata>();
|
||||
FPODReader.importFile(cpFile, fpodData, from, maxNum);
|
||||
|
||||
ArrayList<CPODClick> cpodClickData = new ArrayList<CPODClick>();
|
||||
|
||||
CPODClick cpodClick;
|
||||
for (int i=0; i<fpodData.size(); i++) {
|
||||
//how many samples are we into the clicks
|
||||
long fileSamples = (long) (((fpodData.get(i).minute*60) + (fpodData.get(i).FiveMusec*5/1000000.))*CPODClickDataBlock.CPOD_SR);
|
||||
|
||||
cpodClick = CPODClick.makeFPODClick(fpodData.get(i).getTimeMillis(), fileSamples, fpodData.get(i));
|
||||
|
||||
cpodClickData.add(cpodClick);
|
||||
}
|
||||
|
||||
return cpodClickData;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Import an FPOD file.
|
||||
|
||||
@ -2,6 +2,7 @@ package cpod.fx;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
import javax.swing.SwingUtilities;
|
||||
@ -98,7 +99,7 @@ public class CPODSettingsPane extends SettingsPane<CPODParams> {
|
||||
private PamButton importButton;
|
||||
|
||||
|
||||
private List<Task<Integer>> tasks;
|
||||
private Task<Integer> task;
|
||||
|
||||
private boolean running = false;
|
||||
|
||||
@ -136,7 +137,7 @@ public class CPODSettingsPane extends SettingsPane<CPODParams> {
|
||||
browsFileButton.setTooltip(new Tooltip("Browse to select individual or mutliple CP1 or CP3 files or FP1 or FP3 files"));
|
||||
browsFileButton.setOnAction((action)->{
|
||||
|
||||
List<File> files = fileChooser.showOpenMultipleDialog(this.getFXWindow());
|
||||
List<File> files = new LinkedList<File>(fileChooser.showOpenMultipleDialog(this.getFXWindow()));
|
||||
setNewFiles(files);
|
||||
|
||||
});
|
||||
@ -296,54 +297,17 @@ public class CPODSettingsPane extends SettingsPane<CPODParams> {
|
||||
|
||||
//begins the import
|
||||
// this.tasks = this.cpodControl.getCpodImporter().importCPODData(files);
|
||||
this.tasks = this.cpodControl.importPODData(files);
|
||||
this.task = this.cpodControl.importPODData(files);
|
||||
|
||||
if (tasks ==null) return false;
|
||||
if (task ==null) return false;
|
||||
|
||||
this.progressBar.setProgress(-1.);
|
||||
|
||||
this.progressBar.progressProperty().bind(tasks.get(0).progressProperty());
|
||||
this.progressLabel.textProperty().bind(tasks.get(0).messageProperty());
|
||||
|
||||
|
||||
//binds the progress bar - imports are
|
||||
for (int i=0; i<tasks.size()-1; i++) {
|
||||
final int ii = i;
|
||||
//Will be called if the tasks are cancelled or succeed.
|
||||
tasks.get(ii).setOnCancelled((worker)->{
|
||||
this.progressBar.progressProperty().bind(tasks.get(ii+1).progressProperty());
|
||||
this.progressLabel.textProperty().bind(tasks.get(ii+1).messageProperty());
|
||||
});
|
||||
|
||||
tasks.get(ii).setOnSucceeded((worker)->{
|
||||
System.out.println("IMPORT FINISHED: " + ii);
|
||||
this.progressBar.progressProperty().bind(tasks.get(ii+1).progressProperty());
|
||||
this.progressLabel.textProperty().bind(tasks.get(ii+1).messageProperty());
|
||||
});
|
||||
|
||||
tasks.get(ii).setOnFailed((worker)->{
|
||||
this.progressBar.progressProperty().bind(tasks.get(ii+1).progressProperty());
|
||||
this.progressLabel.textProperty().bind(tasks.get(ii+1).messageProperty());
|
||||
});
|
||||
}
|
||||
|
||||
tasks.get(tasks.size()-1).setOnCancelled((worker)->{
|
||||
importingFinished();
|
||||
});
|
||||
|
||||
tasks.get(tasks.size()-1).setOnSucceeded((worker)->{
|
||||
importingFinished();
|
||||
});
|
||||
|
||||
|
||||
tasks.get(tasks.size()-1).setOnFailed((worker)->{
|
||||
importingFinished();
|
||||
});
|
||||
|
||||
|
||||
this.progressBar.progressProperty().bind(task.progressProperty());
|
||||
this.progressLabel.textProperty().bind(task.messageProperty());
|
||||
|
||||
//run the tasks
|
||||
this.cpodControl.getCpodImporter().runTasks(tasks);
|
||||
this.cpodControl.getCpodImporter().runTasks(task);
|
||||
|
||||
return true;
|
||||
}
|
||||
@ -383,11 +347,7 @@ public class CPODSettingsPane extends SettingsPane<CPODParams> {
|
||||
* Stop the import.
|
||||
*/
|
||||
private void stopImport() {
|
||||
if (tasks!=null) {
|
||||
for (int i=0; i<tasks.size(); i++) {
|
||||
tasks.get(i).cancel();
|
||||
}
|
||||
}
|
||||
task.cancel();
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Loading…
Reference in New Issue
Block a user