nordic-dev.net/PAMGuard
nordic-dev.net/PAMGuard
2
0
Fork 0
mirror of https://github.com/PAMGuard/PAMGuard.git synced 2024-11-25 08:32:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Working on the exporter

Browse Source
This commit is contained in:
Jamie Mac 2024-06-13 08:56:25 +01:00
parent c607f17449
commit c1bb771cff
11 changed files with 271 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/dataPlotsFX/overlaymark/menuOptions/RExportOverlayMenu.java
View File

@ -44,7 +44,7 @@ public class RExportOverlayMenu extends ExportOverlayMenu {
* Create the export overlay
*/
public RExportOverlayMenu(){
rExportManger= new RExportManager();
rExportManger= new RExportManager(null);
buttonNode = createButton();

9
src/export/CSVExport/CSVExportManager.java
View File

@ -6,6 +6,10 @@ import java.util.List;
import PamguardMVC.PamDataUnit;
import export.PamDataUnitExporter;
/**
* Export to CSV files which are RAVEN compatible.
*/
public class CSVExportManager implements PamDataUnitExporter{
@Override
@ -41,4 +45,9 @@ public class CSVExportManager implements PamDataUnitExporter{
}
@Override
public boolean isNeedsNewFile() {
return false;
}
}

6
src/export/MLExport/MLDetectionsManager.java
View File

@ -269,6 +269,12 @@ public class MLDetectionsManager implements PamDataUnitExporter {
}
@Override
public boolean isNeedsNewFile() {
// TODO Auto-generated method stub
return false;
}

12
src/export/PamDataUnitExporter.java
View File

@ -41,15 +41,21 @@ public interface PamDataUnitExporter {
public String getIconString();
/**
* Get the name of the exporter
* @return
* Get the name of the exporter.
* @return the name of the exporter.
*/
public String getName();
/**
* Clsoe the exporter
* Close the exporter.
*/
public void close();
/**
* Check whether and exporter needs a new file
* @return true if we need a new file.
*/
public boolean isNeedsNewFile();
}

21
src/export/PamExporterManager.java
View File

@ -56,11 +56,12 @@ public class PamExporterManager {
"yyyy_MM_dd_HHmmss");
public PamExporterManager() {
pamExporters = new ArrayList<PamDataUnitExporter>();
//add the MATLAB export
pamExporters.add(new MLDetectionsManager());
pamExporters.add(new RExportManager());
pamExporters.add(new RExportManager(this));
pamExporters.add(new WavFileExportManager());
pamExporters.add(new CSVExportManager());
}
@ -74,7 +75,7 @@ public class PamExporterManager {
if (dataUnit==null) {
if (force) {
System.out.println("Write data 1!!" + dataUnitBuffer.size() );
// System.out.println("Write data 1!!" + dataUnitBuffer.size() );
//finish off saving any buffered data
exportOK = pamExporters.get(exportParams.exportChoice).exportData(currentFile, dataUnitBuffer, true);
dataUnitBuffer.clear();
@ -83,10 +84,10 @@ public class PamExporterManager {
}
//if file is null or too large create another a file for saving.
if (currentFile == null || isFileSizeMax(currentFile)) {
if (currentFile == null || isNeedsNewFile(currentFile, pamExporters.get(exportParams.exportChoice))) {
Date date = new Date(dataUnit.getTimeMilliseconds());
String newFileName = "PAM_" + dataFormat.format(date);
String newFileName = "PAM_" + dataFormat.format(date) + dataUnit.getParentDataBlock().getDataName().replace(" ", "_");
//create a new file - note each exporter is responsible for closing the file after writing
//so previous files should already be closed
@ -98,10 +99,10 @@ public class PamExporterManager {
dataUnitBuffer.add(dataUnit);
System.out.println("Write data unit " + dataUnitBuffer.size() + " to: "+ currentFile);
// System.out.println("Write data unit " + dataUnitBuffer.size() + " to: "+ currentFile);
if (dataUnitBuffer.size()>=BUFFER_SIZE || force) {
System.out.println("Write data 2!!" + dataUnitBuffer.size());
// System.out.println("Write data 2!!" + dataUnitBuffer.size());
exportOK = pamExporters.get(exportParams.exportChoice).exportData(currentFile, dataUnitBuffer, true);
dataUnitBuffer.clear();
}
@ -118,10 +119,14 @@ public class PamExporterManager {
/**
* Check whether the current file is greater than the maximum allowed file size.
* @param currentFile2 - the current file
* @param pamDataUnitExporter
* @return true of greater than or equal to the maximum file size.
*/
private boolean isFileSizeMax(File currentFile2) {
return getFileSizeMegaBytes(currentFile2) >= MAX_FILE_SIZE_MB;
private boolean isNeedsNewFile(File currentFile2, PamDataUnitExporter pamDataUnitExporter) {
if( getFileSizeMegaBytes(currentFile2) >= exportParams.maximumFileSize) {
return true;
};
return pamDataUnitExporter.isNeedsNewFile();
}
/**

26
src/export/RExport/RExportManager.java
View File

@ -18,6 +18,7 @@ import org.renjin.sexp.PairList.Builder;
import PamUtils.PamArrayUtils;
import PamguardMVC.PamDataUnit;
import export.PamDataUnitExporter;
import export.PamExporterManager;
import export.MLExport.MLDetectionsManager;
/**
@ -29,7 +30,6 @@ import export.MLExport.MLDetectionsManager;
public class RExportManager implements PamDataUnitExporter {
/**
*
* All the possible RDataUnit export classes.
*/
ArrayList<RDataUnitExport> rDataExport = new ArrayList<RDataUnitExport>();
@ -39,8 +39,11 @@ public class RExportManager implements PamDataUnitExporter {
private Builder allData;
private PamExporterManager pamExporterManager;
public RExportManager(){
public RExportManager(PamExporterManager pamExporterManager){
this.pamExporterManager=pamExporterManager;
/***Add more options here to export data units****/
rDataExport.add(new RClickExport());
rDataExport.add(new RWhistleExport());
@ -55,17 +58,14 @@ public class RExportManager implements PamDataUnitExporter {
* Note - there is no way to save data units to R files wothout loading the file into memory.
* So everything is stored in memory until saved.
*/
// then
PamDataUnit minByTime = PamArrayUtils.getMinTimeMillis(dataUnits);
//matlab struct must start with a letter.
//MATLAB struct must start with a letter.
Date date = new Date(minByTime.getTimeMilliseconds());
String entryName = "det_" + MLDetectionsManager.dataFormat.format( date);
// System.out.println("Save R data! "+ dataUnits.size());
// System.out.println("Export R file!!" + dataUnits.size());
//is there an existing writer? Is that writer writing to the correct file?
if (allData==null || !fileName.equals(currentFileName)) {
@ -259,5 +259,19 @@ public class RExportManager implements PamDataUnitExporter {
}
@Override
public boolean isNeedsNewFile() {
//Rdata can't be appended to a file so we cannot check file sizes.
// pamExporterManager.getExportParams().maximumFileSize;
//TODO
//check file size against the export params.
System.out.println("RData length: " + allData.length());
return false;
}
}

174
src/export/swing/ExportProcessDialog.java
View File

@ -9,7 +9,6 @@ import java.awt.Window;
import java.awt.event.ActionListener;
import java.io.File;
import java.util.ArrayList;
import javax.swing.BoxLayout;
import javax.swing.ButtonGroup;
import javax.swing.JFileChooser;
@ -38,7 +37,6 @@ import export.PamExporterManager;
import export.layoutFX.ExportParams;
import offlineProcessing.OLProcessDialog;
import offlineProcessing.OfflineTaskGroup;
import offlineProcessing.TaskStatus;
/**
* Handles an offline dialog for processing offline data and exporting to bespoke file types.
@ -129,6 +127,9 @@ public class ExportProcessDialog {
*/
private JFileChooser fc;
/**
* S hows the folder stuff is going to export to.
*/
private JTextField exportTo;
/**
@ -268,7 +269,7 @@ public class ExportProcessDialog {
Ikon icon = null;
/**
* This is nasty but we won't have many exporters and this is the only
* This is NASTY but we won't have many exporters and this is the only
* good way to get this to work in Swing.
*/
switch (iconString) {
@ -353,11 +354,10 @@ public class ExportProcessDialog {
}
class ExportTaskGroup extends OfflineTaskGroup{
class ExportTaskGroup extends OfflineTaskGroup {
public ExportTaskGroup(String settingsName) {
super(null, settingsName);
// TODO Auto-generated constructor stub
}
@ -365,6 +365,170 @@ public class ExportProcessDialog {
public String getUnitType() {
return "Export Data";
}
/**
* Override the tasks o it runs through all tasks for each datablock. Usually
* task groups deal with just one parent datablock but exporters export from
* different data blocks. The only way to deal with this is to let the task run
* again and again through all tasks and letting tasks themselves check the
* correct data units are being exported.
*/
@Override
public boolean runTasks() {
boolean OK = true;
for (int i=0; i<getNTasks(); i++) {
this.setPrimaryDataBlock(getTask(i).getDataBlock());
super.runTasks();
}
return OK;
}
//
//
// int nDatas = primaryDataBlock.getUnitsCount();
// int nSay = Math.max(1, nDatas / 100);
//// int nDone = 0;
// int nTasks = getNTasks();
// PamDataUnit dataUnit;
// OfflineTask aTask;
// boolean unitChanged;
// DataUnitFileInformation fileInfo;
// String dataName;
// if (mapPoint != null) {
// dataName = mapPoint.getName();
// }
// else {
// dataName = "Loaded Data";
// }
// /**
// * Make sure that any data from required data blocks is loaded. First check the
// * start and end times of the primary data units we actually WANT to process
// * Also get a count of found data - may be able to leave without having to do anything at all
// */
// ListIterator<PamDataUnit> it = primaryDataBlock.getListIterator(0);
// long procDataStart = Long.MAX_VALUE;
// long procDataEnd = 0;
// int nToProcess = 0;
// while (it.hasNext()) {
// dataUnit = it.next();
// /**
// * Make sure we only process data units within the current time interval.
// */
// if (dataUnit.getTimeMilliseconds() < processStartTime) {
// continue;
// }
// if (dataUnit.getTimeMilliseconds() > processEndTime) {
// break;
// }
//// if (shouldProcess(dataUnit) == false) {
//// continue;
//// }
// procDataStart = Math.min(procDataStart, dataUnit.getTimeMilliseconds());
// procDataEnd = Math.max(procDataEnd, dataUnit.getEndTimeInMilliseconds());
// // do this one too - just to make sure in case end time returns zero.
// procDataEnd = Math.max(procDataEnd, dataUnit.getTimeMilliseconds());
// nToProcess++; // increase toprocess counter
// }
// if (nToProcess == 0) {
// return;
// }
// PamDataBlock aDataBlock;
// RequiredDataBlockInfo blockInfo;
// /*
// * if the data interval is < 1 hour, then load it all now
// * otherwise we'll do it on a data unit basis.
// */
////// long maxSecondaryLoad = 1800L*1000L;
////// if (procDataEnd - procDataStart < maxSecondaryLoad) {
//// loadSecondaryData(procDataStart, procDataEnd);
////// }
// // remember the end time of the data so we can use the "new data" selection flag.
// taskGroupParams.lastDataTime = Math.min(primaryDataBlock.getCurrentViewDataEnd(),processEndTime);
// // synchronized(primaryDataBlock) {
// /*
// * Call newDataLoaded for each task before getting on with processing individual data units.
// */
//
// /**
// * Now process the data
// */
// it = primaryDataBlock.getListIterator(0);
// unitChanged = false;
// int totalUnits = 0;
// int unitsChanged = 0;
// boolean doTasks = false;
// while (it.hasNext()) {
// dataUnit = it.next();
// totalUnits++;
// doTasks = true;
// /**
// * Make sure we only process data units within the current time interval.
// */
// if (dataUnit.getTimeMilliseconds() < processStartTime) {
// continue;
// }
// if (dataUnit.getTimeMilliseconds() > processEndTime) {
// break;
// }
//
// if (shouldProcess(dataUnit) == false) {
// doTasks = false;
// }
//
// if (doTasks) {
// /*
// * load the secondary datablock data. this can be called even if
// * it was called earlier on since it wont' reload if data are already
// * in memory.
// */
//// loadSecondaryData(dataUnit.getTimeMilliseconds(), dataUnit.getEndTimeInMilliseconds());
//
// for (int iTask = 0; iTask < nTasks; iTask++) {
// aTask = getTask(iTask);
// if (aTask.isDoRun() == false || !isInTimeChunk(dataUnit, taskGroupParams.timeChunks)) {
// continue;
// }
// cpuMonitor.start();
// unitChanged |= aTask.processDataUnit(dataUnit);
// cpuMonitor.stop();
// }
// if (unitChanged) {
// fileInfo = dataUnit.getDataUnitFileInformation();
// if (fileInfo != null) {
// fileInfo.setNeedsUpdate(true);
// }
// dataUnit.updateDataUnit(System.currentTimeMillis());
// }
// dataUnit.freeData();
// }
// if (instantKill) {
// break;
// }
// unitsChanged++;
// if (totalUnits%nSay == 0) {
// publish(new TaskMonitorData(TaskStatus.RUNNING, TaskActivity.PROCESSING, nToProcess, totalUnits, dataName,
// dataUnit.getTimeMilliseconds()));
// }
// }
// for (int iTask = 0; iTask < nTasks; iTask++) {
// aTask = getTask(iTask);
// if (aTask.isDoRun() == false) {
// continue;
// }
// aTask.loadedDataComplete();
// }
// // }
// publish(new TaskMonitorData(TaskStatus.RUNNING, TaskActivity.SAVING, nToProcess, totalUnits, dataName,
// processEndTime));
// for (int i = 0; i < affectedDataBlocks.size(); i++) {
// //System.out.println("SAVE VIEWER DATA FOR: " + affectedDataBlocks.get(i) );
// aDataBlock = affectedDataBlocks.get(i);
// aDataBlock.saveViewerData();
// }
// Debug.out.printf("Processd %d out of %d data units at " + mapPoint + "\n", unitsChanged, totalUnits);
// commitDatabase();
// }
}

14
src/export/swing/ExportTask.java
View File

@ -32,7 +32,7 @@ public class ExportTask extends OfflineTask<PamDataUnit<?,?>>{
public ExportTask(PamDataBlock<PamDataUnit<?, ?>> parentDataBlock, PamExporterManager exporter) {
super(parentDataBlock);
this.exporter = exporter;
dataSelector=parentDataBlock.getDataSelectCreator().getDataSelector(this.getUnitName() +"_clicks", false, null);
dataSelector=parentDataBlock.getDataSelectCreator().getDataSelector(this.getUnitName() +"_export", false, null);
}
@ -44,23 +44,29 @@ public class ExportTask extends OfflineTask<PamDataUnit<?,?>>{
@Override
public boolean processDataUnit(PamDataUnit<?, ?> dataUnit) {
System.out.println("Huh? " + this.getDataBlock().getDataName() + " " + dataUnit + " " + dataUnit.getParentDataBlock().equals(this.getDataBlock()));
if (dataUnit.getParentDataBlock().equals(this.getDataBlock())) {
//this is very important because the way the expoeter works is that it iterates through multiple parent data blocks
System.out.println(this.getDataBlock().getDataName() + " " + dataUnit);
if (dataSelector==null) exporter.exportDataUnit(dataUnit, false);
else if (dataSelector.scoreData(dataUnit)>0) {
exporter.exportDataUnit(dataUnit, false);
}
}
return false; //we don't need to indicate that anything has changed - we are just exporting.
}
@Override
public void newDataLoad(long startTime, long endTime, OfflineDataMapPoint mapPoint) {
// TODO Auto-generated method stub
// System.out.println("EXPORTER: new data load");
// System.out.println("EXPORTER: new data load");
}
@Override
public void loadedDataComplete() {
System.out.println("EXPORTER: loaded data complete");
// System.out.println("EXPORTER: loaded data complete");
//force the exporter so save any renaming data units in the buffer
exporter.exportDataUnit(null, true);
exporter.close();

7
src/export/wavExport/WavFileExportManager.java
View File

@ -465,6 +465,13 @@ public class WavFileExportManager implements PamDataUnitExporter {
@Override
public boolean isNeedsNewFile() {
return false;
}
// hello(){

7
src/offlineProcessing/OfflineTaskGroup.java
View File

@ -142,7 +142,7 @@ public class OfflineTaskGroup implements PamSettings {
private ArrayList<OfflineTask> offlineTasks = new ArrayList<OfflineTask>();
private TaskGroupWorker worker;
protected TaskGroupWorker worker;
private OfflineSuperDetFilter superDetectionFilter;
@ -377,7 +377,7 @@ public class OfflineTaskGroup implements PamSettings {
* @author Doug Gillespie
*
*/
class TaskGroupWorker extends SwingWorker<Integer, TaskMonitorData> implements ViewLoadObserver {
public class TaskGroupWorker extends SwingWorker<Integer, TaskMonitorData> implements ViewLoadObserver {
volatile boolean instantKill = false;
@ -437,6 +437,7 @@ public class OfflineTaskGroup implements PamSettings {
/**
* Process all data for a list of time chunks. This is robust to the list
* not being in chronological order.
*
* @param timeChunks - the time chunks.
*/
private void processAllData(ArrayList<long[]> timeChunks){
@ -976,4 +977,6 @@ public class OfflineTaskGroup implements PamSettings {
offlineTasks.clear();
}
}

8
src/whistlesAndMoans/WhistleBinaryDataSource.java
View File

@ -151,11 +151,15 @@ public class WhistleBinaryDataSource extends BinaryDataSource {
/**
* Bit of mess sorted out on 15/5/2020. Was working because module version went from 1 to 2 at same time
* as file version went from 3 to 4. May have been some middly stuff where file version and module
* as file version went from 3 to 4. May have been some middle stuff where file version and module
* There is some FV 3 with MV 1, in which case data were probably duplicated.
*/
if (fileVersion > 3) { // basic data now in standard format.
firstSliceSample = startSample = binaryObjectData.getDataUnitBaseData().getStartSample();
if (binaryObjectData.getDataUnitBaseData().getStartSample()==null) {
//some very rare circumstances
firstSliceSample =0;
}
else firstSliceSample = startSample = binaryObjectData.getDataUnitBaseData().getStartSample();
// if the DataUnitBaseData contains a sequence map, use it in place of the channel map
if (binaryObjectData.getDataUnitBaseData().getSequenceBitmap()!=null) {