nordic-dev.net/PAMGuard
nordic-dev.net/PAMGuard
2
0
Fork 0
mirror of https://github.com/PAMGuard/PAMGuard.git synced 2024-11-22 07:02:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added bin granularity

Browse Source 
binned granularity in place. TODO Encounter granularity.
This commit is contained in:
Douglas Gillespie 2023-09-11 17:07:15 +01:00
parent baca001ed8
commit c221d78aa2
25 changed files with 751 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/PamController/settings/output/xml/PamguardXMLWriter.java
View File

@ -581,10 +581,13 @@ public class PamguardXMLWriter implements PamSettings {
return el;
}
private Element writeObjectData(Document doc, Element el, Object data, ArrayList<Object> objectHierarchy) {
public Element writeObjectData(Document doc, Element el, Object data, ArrayList<Object> objectHierarchy) {
if (data == null) {
return null;
}
if (objectHierarchy == null) {
objectHierarchy = new ArrayList<>();
}
if (objectHierarchy.contains(data)) {
// just write the reference, but nothing else or we'll end up in an infinite loop of objects.
Element e = doc.createElement("Object");

5
src/PamguardMVC/PamDataBlock.java
View File

@ -50,6 +50,7 @@ import org.w3c.dom.Element;
import Acquisition.AcquisitionControl;
import Acquisition.AcquisitionProcess;
import pamScrollSystem.ViewLoadObserver;
import tethys.TethysControl;
import tethys.pamdata.AutoTethysProvider;
import tethys.pamdata.TethysDataProvider;
import tethys.species.DataBlockSpeciesManager;
@ -3089,9 +3090,9 @@ public class PamDataBlock<Tunit extends PamDataUnit> extends PamObservable {
* to be bespoke, but for now will autogenerate based on the SQLLogging information.
* @return the tethysDataProvider
*/
public TethysDataProvider getTethysDataProvider() {
public TethysDataProvider getTethysDataProvider(TethysControl tethysControl) {
if (tethysDataProvider == null && PamDetection.class.isAssignableFrom(unitClass) && getLogging() != null) {
tethysDataProvider = new AutoTethysProvider(this);
tethysDataProvider = new AutoTethysProvider(tethysControl, this);
}
return tethysDataProvider;
}

62
src/tethys/TethysControl.java
View File

@ -211,7 +211,7 @@ public class TethysControl extends PamControlledUnit implements PamSettings, Tet
ArrayList<PamDataBlock> sets = new ArrayList<>();
ArrayList<PamDataBlock> allDataBlocks = PamController.getInstance().getDataBlocks();
for (PamDataBlock aDataBlock : allDataBlocks) {
if (aDataBlock.getTethysDataProvider() != null) {
if (aDataBlock.getTethysDataProvider(this) != null) {
sets.add(aDataBlock);
}
}
@ -335,31 +335,31 @@ public class TethysControl extends PamControlledUnit implements PamSettings, Tet
return tethysExportParams;
}
/**
* We'll probably want to
* @param parentFrame
*/
protected void tethysExport(JFrame parentFrame) {
TethysExportParams newExportParams = TethysExportDialog.showDialog(parentFrame, this);
if (newExportParams != null) {
// dialog returns null if cancel was pressed.
tethysExportParams = newExportParams;
exportTethysData(tethysExportParams);
}
}
/**
* We'll arrive here if the dialog has been opened and we want to export Tethys data.
* @param tethysExportParams2
*/
private void exportTethysData(TethysExportParams tethysExportParams) {
TethysExporter tethysExporter = new TethysExporter(this, tethysExportParams);
tethysExporter.doExport();
sendStateUpdate(new TethysState(StateType.TRANSFERDATA));
countProjectDetections();
sendStateUpdate(new TethysState(StateType.NEWPAMGUARDSELECTION));
}
// /**
// * We'll probably want to
// * @param parentFrame
// */
// protected void tethysExport(JFrame parentFrame) {
// TethysExportParams newExportParams = TethysExportDialog.showDialog(parentFrame, this);
// if (newExportParams != null) {
// // dialog returns null if cancel was pressed.
// tethysExportParams = newExportParams;
// exportTethysData(tethysExportParams);
// }
// }
//
// /**
// * We'll arrive here if the dialog has been opened and we want to export Tethys data.
// * @param tethysExportParams2
// */
// private void exportTethysData(TethysExportParams tethysExportParams) {
// TethysExporter tethysExporter = new TethysExporter(this, tethysExportParams);
// tethysExporter.doExport();
//
// sendStateUpdate(new TethysState(StateType.TRANSFERDATA));
// countProjectDetections();
// sendStateUpdate(new TethysState(StateType.NEWPAMGUARDSELECTION));
// }
/**
* Get global deployment data. This is a bit of a mess, trying to use a separate module
@ -644,4 +644,14 @@ public class TethysControl extends PamControlledUnit implements PamSettings, Tet
return itisFunctions;
}
/**
* Called when a detections document has been exported.
* @param dataBlock
*/
public void exportedDetections(PamDataBlock dataBlock) {
sendStateUpdate(new TethysState(StateType.TRANSFERDATA));
countProjectDetections();
sendStateUpdate(new TethysState(StateType.NEWPAMGUARDSELECTION));
}
}

21
src/tethys/dbxml/DBXMLQueries.java
View File

@ -541,6 +541,27 @@ public class DBXMLQueries {
// String queryBase = "{\"return\":[\"Deployment/Project\"],\"select\":[],\"enclose\":1}";
}
/**
* Find out if a document exists ?
* @param collection
* @param documentId
* @return
*/
public boolean documentExists(String collection, String documentId) {
Queries queries = dbXMLConnect.getTethysQueries();
String result = null;
try {
result = queries.getDocument(collection, documentId);
} catch (Exception e) {
return false;
}
if (result == null || result.length() == 0) {
return false;
}
return result.contains(documentId);
}
/**
* Count on effort detections in a Detections document
* @param docName

161
src/tethys/detection/BinnedGranularityHandler.java Normal file
View File

@ -0,0 +1,161 @@
package tethys.detection;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Set;
import PamguardMVC.PamDataBlock;
import PamguardMVC.PamDataUnit;
import nilus.Detection;
import nilus.SpeciesIDType;
import tethys.TethysControl;
import tethys.TethysTimeFuncs;
import tethys.output.StreamExportParams;
import tethys.output.TethysExportParams;
import tethys.pamdata.TethysDataProvider;
import tethys.species.DataBlockSpeciesCodes;
import tethys.species.DataBlockSpeciesManager;
import tethys.species.DataBlockSpeciesMap;
import tethys.species.SpeciesMapItem;
/**
* Binned granularity
* Will have to collect different counts for each type of call for each datablock (if there
* are such things) so a little more complicated than might be expected.
* @author dg50
*
*/
public class BinnedGranularityHandler extends GranularityHandler {
private double binDurationSeconds;
private long binStartMillis, binEndMillis;
private TethysDataProvider dataProvider;
private DataBlockSpeciesManager speciesManager;
private HashMap<String, Detection> currentDetections;
public BinnedGranularityHandler(TethysControl tethysControl, PamDataBlock dataBlock,
TethysExportParams tethysExportParams, StreamExportParams streamExportParams) {
super(tethysControl, dataBlock, tethysExportParams, streamExportParams);
binDurationSeconds = streamExportParams.binDurationS;
dataProvider = dataBlock.getTethysDataProvider(tethysControl);
speciesManager = dataBlock.getDatablockSpeciesManager();
currentDetections = new HashMap<String, Detection>();
}
@Override
public void prepare(long timeMillis) {
long binStart = DetectionsHandler.roundDownBinStart(timeMillis, (long) (binDurationSeconds*1000));
startBin(binStart);
}
private void startBin(long timeMillis) {
binStartMillis = timeMillis;
binEndMillis = binStartMillis + (long) (binDurationSeconds*1000.);
/*
* now make a Detection object for every possible species that
* this might throw out.
*/
ArrayList<String> speciesCodes = speciesManager.getAllSpeciesCodes();
String defaultCode = speciesManager.getDefaultSpeciesCode();
Detection det;
currentDetections.put(defaultCode, det = new Detection());
det.setStart(TethysTimeFuncs.xmlGregCalFromMillis(binStartMillis));
det.setEnd(TethysTimeFuncs.xmlGregCalFromMillis(binEndMillis));
det.setCount(BigInteger.ZERO);
det.setChannel(BigInteger.ZERO);
// add codes at end, just before output.
if (speciesCodes != null) {
for (String code : speciesCodes) {
currentDetections.put(code, det = new Detection());
det.setStart(TethysTimeFuncs.xmlGregCalFromMillis(binStartMillis));
det.setEnd(TethysTimeFuncs.xmlGregCalFromMillis(binEndMillis));
det.setCount(BigInteger.ZERO);
det.setChannel(BigInteger.ZERO);
}
}
}
@Override
public Detection[] addDataUnit(PamDataUnit dataUnit) {
Detection[] detections = null;
if (dataUnit.getTimeMilliseconds() >= binEndMillis) {
detections = closeBins(dataUnit.getTimeMilliseconds());
}
String speciesCode = speciesManager.getSpeciesCode(dataUnit);
Detection det = currentDetections.get(speciesCode);
if (det != null) {
/*
* Increase the detection count
*/
int count = det.getCount().intValue();
count++;
det.setCount(BigInteger.valueOf(count));
/*
* Add to the channel map too ...
*/
int channel = det.getChannel().intValue();
channel |= dataUnit.getChannelBitmap();
det.setChannel(BigInteger.valueOf(channel));
}
return detections;
}
/**
* Called when units arrive after end of current bin, and also
* at end of deployment output, to get that last bine.
* @param timeMilliseconds
* @return
*/
private Detection[] closeBins(long timeMilliseconds) {
Set<String> speciesKeys = currentDetections.keySet();
int n = speciesKeys.size();
int nGood = 0;
DataBlockSpeciesMap speciesMap = speciesManager.getDatablockSpeciesMap();
Detection detections[] = new Detection[n];
for (String key : speciesKeys) {
Detection det = currentDetections.get(key);
int callCount = det.getCount().intValue();
if (callCount < Math.max(streamExportParams.minBinCount,1)) {
continue;
}
SpeciesMapItem speciesStuff = speciesMap.getItem(key); // should be non null!
if (speciesStuff == null) {
continue;
}
SpeciesIDType species = new SpeciesIDType();
species.setValue(BigInteger.valueOf(speciesStuff.getItisCode()));
det.setSpeciesId(species);
if (speciesStuff.getCallType() != null) {
det.getCall().add(speciesStuff.getCallType());
}
detections[nGood++] = det;
}
// finally, start new bins (not really needed on last call, but do anyway).
startBin(binEndMillis);
/*
* Clean up the end of the array and return detections that have enough calls.
*/
if (nGood == 0) {
return null;
}
detections = Arrays.copyOf(detections, nGood);
return detections;
}
@Override
public Detection[] cleanup(long timeMillis) {
return closeBins(timeMillis);
}
}

40
src/tethys/detection/CallGranularityHandler.java Normal file
View File

@ -0,0 +1,40 @@
package tethys.detection;
import PamguardMVC.PamDataBlock;
import PamguardMVC.PamDataUnit;
import nilus.Detection;
import tethys.TethysControl;
import tethys.output.StreamExportParams;
import tethys.output.TethysExportParams;
import tethys.pamdata.TethysDataProvider;
public class CallGranularityHandler extends GranularityHandler {
private TethysDataProvider dataProvider;
public CallGranularityHandler(TethysControl tethysControl, PamDataBlock dataBlock,
TethysExportParams tethysExportParams, StreamExportParams streamExportParams) {
super(tethysControl, dataBlock, tethysExportParams, streamExportParams);
dataProvider = dataBlock.getTethysDataProvider(tethysControl);
}
@Override
public void prepare(long timeMillis) {
// never anything to do here for call level granularity.
}
@Override
public Detection[] addDataUnit(PamDataUnit dataUnit) {
Detection det = dataProvider.createDetection(dataUnit, tethysExportParams, streamExportParams);
return toDetectionArray(det);
}
@Override
public Detection[] cleanup(long timeMillis) {
// never anything to do here for call level granularity.
return null;
}
}

34
src/tethys/detection/DetectionGranularity.java
View File

@ -1,34 +0,0 @@
package tethys.detection;
/**
* Class to help define what the granularity of exported detections
* documents should be. The entire document will be in memory, so it
* may be necessary to add many detections documents into the database
* for a single Deployment.
* @author dg50
*
*/
public class DetectionGranularity {
public enum GRANULARITY {NONE, BINARYFILE, TIME};
/**
* Type of granularity. Are data all in one lump, split by binary file or by time.
*/
GRANULARITY granularity = GRANULARITY.NONE;
/**
* Granularity interval in seconds. Output system will try to round these
* to something with sensible boundaries.
* This field is only needed when using the GRANULARITY.TIME option.
*/
public long granularityIntervalSeconds;
public DetectionGranularity(GRANULARITY granularity, long granularityIntervalSeconds) {
super();
this.granularity = granularity;
this.granularityIntervalSeconds = granularityIntervalSeconds;
}
}

90
src/tethys/detection/DetectionsHandler.java
View File

@ -1,33 +1,20 @@
package tethys.detection;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
import javax.swing.SwingWorker;
import javax.xml.XMLConstants;
import javax.xml.bind.JAXBException;
import javax.xml.parsers.ParserConfigurationException;
import org.pamguard.x3.sud.SUDClickDetectorInfo;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import PamController.PamControlledUnit;
import PamController.PamguardVersionInfo;
import PamController.settings.output.xml.PamguardXMLWriter;
import PamModel.PamPluginInterface;
import PamUtils.PamCalendar;
import PamUtils.XMLUtils;
import PamguardMVC.PamDataBlock;
import PamguardMVC.PamDataUnit;
import PamguardMVC.PamProcess;
import PamguardMVC.dataSelector.DataSelector;
import dataMap.OfflineDataMap;
import dataMap.OfflineDataMapPoint;
import metadata.deployment.DeploymentData;
import nilus.AlgorithmType;
import nilus.AlgorithmType.Parameters;
import nilus.AlgorithmType.SupportSoftware;
import nilus.DataSourceType;
import nilus.Deployment;
@ -38,21 +25,16 @@ import nilus.DetectionGroup;
import nilus.Detections;
import nilus.Helper;
import tethys.TethysControl;
import tethys.TethysState;
import tethys.TethysState.StateType;
import tethys.deployment.DeploymentHandler;
import tethys.TethysStateObserver;
import tethys.TethysTimeFuncs;
import tethys.dbxml.DBXMLConnect;
import tethys.dbxml.TethysException;
import tethys.detection.DetectionGranularity.GRANULARITY;
import tethys.deployment.DeploymentHandler;
import tethys.niluswraps.PDeployment;
import tethys.niluswraps.PDetections;
import tethys.niluswraps.TethysCollections;
import tethys.output.StreamExportParams;
import tethys.output.TethysExportParams;
import tethys.pamdata.TethysDataPoint;
import tethys.pamdata.TethysDataProvider;
import tethys.swing.export.ExportWorkerCard;
public class DetectionsHandler {
@ -272,7 +254,7 @@ public class DetectionsHandler {
// effort.set // no setter for DetectionEffortKind
List<DetectionEffortKind> effortKinds = effort.getKind();
TethysDataProvider dataProvider = dataBlock.getTethysDataProvider();
TethysDataProvider dataProvider = dataBlock.getTethysDataProvider(tethysControl);
dataProvider.getEffortKinds(pDeployment, effortKinds, exportParams);
@ -363,6 +345,18 @@ public class DetectionsHandler {
activeExport = false;
}
/**
* Round a bin start so that it's aligned correctly with
* day starts.
* @param binStart
* @param binInterval
* @return
*/
public static long roundDownBinStart(long binStart, long binInterval) {
binStart/=binInterval;
return binStart*binInterval;
}
/**
* Export detections in all deployments for this PAMGuard dataset.
* @param dataBlock
@ -381,23 +375,24 @@ public class DetectionsHandler {
ArrayList<PDeployment> deployments = depHandler.getMatchedDeployments();
Detections currentDetections = null;
OfflineDataMap dataMap = dataBlock.getPrimaryDataMap();
TethysDataProvider dataProvider = dataBlock.getTethysDataProvider();
DataSelector dataSelector = dataBlock.getDataSelector(tethysControl.getDataSelectName(), false);
int totalCount = dataMap.getDataCount();
int skipCount = 0;
int exportCount = 0;
long lastUnitTime = 0;
DetectionExportProgress prog;
GranularityHandler granularityHandler = GranularityHandler.getHandler(streamExportParams.granularity, tethysControl, dataBlock, exportParams, streamExportParams);
for (PDeployment deployment : deployments) {
int documentCount = 0;
prog = new DetectionExportProgress(deployment, null,
lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_GATHERING);
exportObserver.update(prog);
granularityHandler.prepare(deployment.getAudioStart());
// export everything in that deployment.
// need to loop through all map points in this interval.
List<OfflineDataMapPoint> mapPoints = dataMap.getMapPoints();
for (OfflineDataMapPoint mapPoint : mapPoints) {
if (activeExport == false) {
if (!activeExport) {
prog = new DetectionExportProgress(deployment, currentDetections,
lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_CANCELED);
exportObserver.update(prog);
@ -418,10 +413,21 @@ public class DetectionsHandler {
skipCount += dataBlock.getUnitsCount() - dataCopy.size();
DetectionGroup onEffort = currentDetections.getOnEffort();
for (PamDataUnit dataUnit : dataCopy) {
Detection det = dataProvider.createDetection(dataUnit, exportParams, streamExportParams);
/*
* Here is where we need to handle the different granularities.
*/
Detection dets[] = granularityHandler.addDataUnit(dataUnit);
if (dets != null) {
for (int dd = 0; dd < dets.length; dd++) {
exportCount++;
documentCount++;
onEffort.getDetection().add(det);
onEffort.getDetection().add(dets[dd]);
}
}
// Detection det = dataProvider.createDetection(dataUnit, exportParams, streamExportParams);
// exportCount++;
// documentCount++;
// onEffort.getDetection().add(det);
lastUnitTime = dataUnit.getTimeMilliseconds();
}
@ -443,7 +449,16 @@ public class DetectionsHandler {
}
}
if (currentDetections != null) {
Detection dets[] = granularityHandler.cleanup(deployment.getAudioEnd());
if (dets != null) {
for (int dd = 0; dd < dets.length; dd++) {
exportCount++;
documentCount++;
currentDetections.getOnEffort().getDetection().add(dets[dd]);
}
}
prog = new DetectionExportProgress(deployment, currentDetections,
lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_WRITING);
closeDetectionsDocument(currentDetections, deployment.getAudioEnd());
@ -472,7 +487,17 @@ public class DetectionsHandler {
}
String prefix = deployment.deployment.getId();
detections.setId(String.format("%s_%d", prefix, uniqueDetectionsId++));
String fullId = "";
/*
* Check the document name isn't already used and increment id as necessary.
*/
while (true) {
fullId = String.format("%s_%d", prefix, uniqueDetectionsId++);
if (!tethysControl.getDbxmlQueries().documentExists(TethysCollections.Detections.toString(), fullId)) {
break;
}
}
detections.setId(fullId);
// detections.setDescription(dataProvider.getDescription(deployment, tethysExportParams));
detections.setDescription(exportParams.getNilusDetectionDescription());
DataSourceType dataSource = new DataSourceType();
@ -481,7 +506,7 @@ public class DetectionsHandler {
detections.setDataSource(dataSource);
AlgorithmType algorithm = detections.getAlgorithm();
TethysDataProvider dataProvider = dataBlock.getTethysDataProvider();
TethysDataProvider dataProvider = dataBlock.getTethysDataProvider(tethysControl);
if (dataProvider != null) {
algorithm = dataProvider.getAlgorithm();
// detections.setAlgorithm(algorithm);
@ -532,14 +557,21 @@ public class DetectionsHandler {
@Override
protected Integer doInBackground() throws Exception {
// eventually need to switch over the four granularity options here.
return exportDetections(dataBlock, exportParams, this);
Integer ans = null;
try {
ans = exportDetections(dataBlock, exportParams, this);
}
catch (Exception e) {
e.printStackTrace();
}
return ans;
}
@Override
protected void done() {
// this.
DetectionExportProgress prog = new DetectionExportProgress(null, null, 0, 0, 0, 0, DetectionExportProgress.STATE_COMPLETE);
tethysControl.exportedDetections(dataBlock);
exportObserver.update(prog);
}

36
src/tethys/detection/EncounterGranularityHandler.java Normal file
View File

@ -0,0 +1,36 @@
package tethys.detection;
import PamguardMVC.PamDataBlock;
import PamguardMVC.PamDataUnit;
import nilus.Detection;
import tethys.TethysControl;
import tethys.output.StreamExportParams;
import tethys.output.TethysExportParams;
public class EncounterGranularityHandler extends GranularityHandler {
public EncounterGranularityHandler(TethysControl tethysControl, PamDataBlock dataBlock,
TethysExportParams tethysExportParams, StreamExportParams streamExportParams) {
super(tethysControl, dataBlock, tethysExportParams, streamExportParams);
// TODO Auto-generated constructor stub
}
@Override
public void prepare(long timeMillis) {
// TODO Auto-generated method stub
}
@Override
public Detection[] addDataUnit(PamDataUnit dataUnit) {
// TODO Auto-generated method stub
return null;
}
@Override
public Detection[] cleanup(long timeMillis) {
// TODO Auto-generated method stub
return null;
}
}

100
src/tethys/detection/GranularityHandler.java Normal file
View File

@ -0,0 +1,100 @@
package tethys.detection;
import PamguardMVC.PamDataBlock;
import PamguardMVC.PamDataUnit;
import nilus.Detection;
import nilus.GranularityEnumType;
import tethys.TethysControl;
import tethys.output.StreamExportParams;
import tethys.output.TethysExportParams;
public abstract class GranularityHandler {
protected TethysControl tethysControl;
protected PamDataBlock dataBlock;
protected TethysExportParams tethysExportParams;
protected StreamExportParams streamExportParams;
/**
* @param tethysControl
* @param dataBlock
* @param tethysExportParams
* @param streamExportParams
*/
public GranularityHandler(TethysControl tethysControl, PamDataBlock dataBlock,
TethysExportParams tethysExportParams, StreamExportParams streamExportParams) {
this.tethysControl = tethysControl;
this.dataBlock = dataBlock;
this.tethysExportParams = tethysExportParams;
this.streamExportParams = streamExportParams;
}
/**
* Prepare to start, passing the start time of the effort
* or of the first time bin for binned granularity types.
* @param timeMillis
*/
public abstract void prepare(long timeMillis);
/**
* Put a data unit into a Detection object. for Call granularity
* this will probably return every time. For binned and encounter
* types this will only return at the end of a bin / encounter
* @param dataUnit
* @return Detection object, but only when ready to be added to Detections
*/
public abstract Detection[] addDataUnit(PamDataUnit dataUnit);
/**
* Called after end end of all data units to get the last bin / encounter. <p>
*
* @param timeMillis end time of effort or last bin in milliseconds.
* @return null for Call granularity, otherwise may be non null for binned or encounter.
*/
public abstract Detection[] cleanup(long timeMillis);
/**
* Convert a single detection to a one element array since that's what'
* most functions need to return.
* @param det
* @return
*/
protected Detection[] toDetectionArray(Detection det) {
if (det == null) {
return null;
}
Detection[] dets = new Detection[1];
dets[0] = det;
return dets;
}
/**
* Create the correct type of granularity handler to put individual data units into
* Detection objects.
* @param granularity
* @param tethysControl
* @param dataBlock
* @param tethysExportParams
* @param streamExportParams
* @return
*/
public static GranularityHandler getHandler(GranularityEnumType granularity, TethysControl tethysControl, PamDataBlock dataBlock,
TethysExportParams tethysExportParams, StreamExportParams streamExportParams) {
switch (granularity) {
case BINNED:
return new BinnedGranularityHandler(tethysControl, dataBlock, tethysExportParams, streamExportParams);
case CALL:
return new CallGranularityHandler(tethysControl, dataBlock, tethysExportParams, streamExportParams);
case ENCOUNTER:
new EncounterGranularityHandler(tethysControl, dataBlock, tethysExportParams, streamExportParams);
case GROUPED:
return new GroupedGranularityHandler(tethysControl, dataBlock, tethysExportParams, streamExportParams);
default:
break;
}
return null;
}
}

36
src/tethys/detection/GroupedGranularityHandler.java Normal file
View File

@ -0,0 +1,36 @@
package tethys.detection;
import PamguardMVC.PamDataBlock;
import PamguardMVC.PamDataUnit;
import nilus.Detection;
import tethys.TethysControl;
import tethys.output.StreamExportParams;
import tethys.output.TethysExportParams;
public class GroupedGranularityHandler extends GranularityHandler {
public GroupedGranularityHandler(TethysControl tethysControl, PamDataBlock dataBlock,
TethysExportParams tethysExportParams, StreamExportParams streamExportParams) {
super(tethysControl, dataBlock, tethysExportParams, streamExportParams);
// TODO Auto-generated constructor stub
}
@Override
public void prepare(long timeMillis) {
// TODO Auto-generated method stub
}
@Override
public Detection[] addDataUnit(PamDataUnit dataUnit) {
// TODO Auto-generated method stub
return null;
}
@Override
public Detection[] cleanup(long timeMillis) {
// TODO Auto-generated method stub
return null;
}
}

2
src/tethys/output/DatablockSynchInfo.java
View File

@ -8,7 +8,7 @@ import tethys.TethysControl;
* All the information needed to populate a table row in the synchronisation table.
* some will need to be set as rarely as possible since it may
* be slow to update. <br>
* This needs to sit alongside the StreamExportParams objects since those others are serialisable wheras
* This needs to sit alongside the StreamExportParams objects since those others are serialisable whereas
* there is a lot of stuff in here which isn't.
* @author dg50
*

6
src/tethys/output/StreamExportParams.java
View File

@ -31,6 +31,12 @@ public class StreamExportParams implements Serializable {
public GranularityEnumType granularity = GranularityEnumType.CALL;
public double binDurationS = 60;
public double encounterGapS = 60;
public int minBinCount = 1;
/*
* Can't have this here since it isn't serializable.
*/

27
src/tethys/output/TethysExporter.java
View File

@ -1,32 +1,16 @@
package tethys.output;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import org.w3c.dom.Document;
import Acquisition.AcquisitionControl;
import Acquisition.AcquisitionProcess;
import Array.ArrayManager;
import Array.Hydrophone;
import Array.PamArray;
import Array.SnapshotGeometry;
import PamController.PamControlledUnit;
import PamController.PamController;
import PamController.PamSettings;
import PamController.settings.output.xml.PamguardXMLWriter;
import PamUtils.PamCalendar;
import PamguardMVC.PamDataBlock;
import PamguardMVC.PamDataUnit;
import PamguardMVC.dataSelector.DataSelector;
import dbxml.uploader.Importer;
import metadata.MetaDataContol;
import metadata.deployment.DeploymentData;
import nilus.Deployment;
@ -35,15 +19,13 @@ import tethys.dbxml.DBXMLConnect;
import tethys.dbxml.TethysException;
import tethys.deployment.DeploymentHandler;
import tethys.deployment.DeploymentOverview;
import tethys.deployment.DeploymentRecoveryPair;
import tethys.deployment.RecordingPeriod;
import tethys.detection.DetectionGranularity;
import tethys.detection.DetectionGranularity.GRANULARITY;
import tethys.detection.DetectionsHandler;
import tethys.pamdata.TethysDataProvider;
import tethys.pamdata.TethysSchema;
/**
* No longer used. This was purely a test class used for making the first couple
* of test connections and exports to Tethys. Can probably delete.
*
* Class sitting at the centre of all operations. It will talk to PAMGuard
* objects to get schemas and data and talk to the database connection to move
* data out (and possibly in). Eventually, a lot of the functionality in here
@ -55,6 +37,7 @@ import tethys.pamdata.TethysSchema;
* @author dg50
*
*/
@Deprecated
public class TethysExporter {
private TethysControl tethysControl;
@ -62,7 +45,7 @@ public class TethysExporter {
private DBXMLConnect dbxmlConnect;
public TethysExporter(TethysControl tethysControl, TethysExportParams tethysExportParams) {
private TethysExporter(TethysControl tethysControl, TethysExportParams tethysExportParams) {
this.tethysControl = tethysControl;
this.tethysExportParams = tethysExportParams;
dbxmlConnect = new DBXMLConnect(tethysControl);

2
src/tethys/output/swing/TethysExportDialog.java
View File

@ -112,7 +112,7 @@ public class TethysExportDialog extends PamDialog {
ArrayList<DataStreamSet> sets = new ArrayList<>();
ArrayList<PamDataBlock> allDataBlocks = PamController.getInstance().getDataBlocks();
for (PamDataBlock aDataBlock : allDataBlocks) {
if (aDataBlock.getTethysDataProvider() != null) {
if (aDataBlock.getTethysDataProvider(tethysControl) != null) {
sets.add(new DataStreamSet(aDataBlock));
}
}

23
src/tethys/pamdata/AutoTethysProvider.java
View File

@ -31,6 +31,7 @@ import nilus.DetectionEffortKind;
import nilus.SpeciesIDType;
import tethys.TethysControl;
import tethys.TethysTimeFuncs;
import tethys.detection.DetectionsHandler;
import tethys.niluswraps.PDeployment;
import tethys.output.StreamExportParams;
import tethys.output.TethysExportParams;
@ -62,8 +63,10 @@ public class AutoTethysProvider implements TethysDataProvider {
private PamDataBlock pamDataBlock;
private PamProcess pamProcess;
private PamControlledUnit pamControlledUnit;
private TethysControl tethysControl;
public AutoTethysProvider(PamDataBlock pamDataBlock) {
public AutoTethysProvider(TethysControl tethysControl, PamDataBlock pamDataBlock) {
this.tethysControl = tethysControl;
this.pamDataBlock = pamDataBlock;
pamProcess = pamDataBlock.getParentProcess();
pamControlledUnit = pamProcess.getPamControlledUnit();
@ -135,7 +138,7 @@ public class AutoTethysProvider implements TethysDataProvider {
Object settings = pamSettings.getSettingsReference();
TethysParameterPacker paramPacker = null;
try {
paramPacker = new TethysParameterPacker();
paramPacker = new TethysParameterPacker(tethysControl);
} catch (JAXBException e) {
// TODO Auto-generated catch block
e.printStackTrace();
@ -425,6 +428,22 @@ public class AutoTethysProvider implements TethysDataProvider {
kind.getSpeciesId().setValue(BigInteger.valueOf(mapItem.getItisCode()));
kind.getGranularity().setValue(exportParams.granularity);
// nilus.DetectionEffortKind.Parameters granularityParams = kind.getParameters();
switch (exportParams.granularity) {
case BINNED:
kind.getGranularity().setBinSizeM(exportParams.binDurationS/60.);
long firstBin = DetectionsHandler.roundDownBinStart(pDeployment.getAudioStart(), (long) (exportParams.binDurationS*1000));
kind.getGranularity().setFirstBinStart(TethysTimeFuncs.xmlGregCalFromMillis(firstBin));
break;
case CALL:
break;
case ENCOUNTER:
kind.getGranularity().setEncounterGapM(exportParams.encounterGapS/60.);
break;
case GROUPED:
break;
}
kind.setCall(mapItem.getCallType());

68
src/tethys/pamdata/TethysParameterPacker.java
View File

@ -14,6 +14,7 @@ import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.dom.DOMResult;
import org.docx4j.model.listnumbering.NumberFormatLowerLetter;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
@ -25,7 +26,10 @@ import PamModel.parametermanager.PamParameterData;
import PamModel.parametermanager.PamParameterSet;
import PamguardMVC.PamDataBlock;
import PamguardMVC.PamProcess;
import PamguardMVC.dataSelector.DataSelectParams;
import PamguardMVC.dataSelector.DataSelector;
import nilus.MarshalXML;
import tethys.TethysControl;
/**
* Functions to pack up a PAMGuard parameters object into the correct format
@ -74,12 +78,15 @@ public class TethysParameterPacker {
private PamguardXMLWriter xmlWriter;
private TethysControl tethysControl;
/**
* @throws JAXBException
*
*/
public TethysParameterPacker() throws JAXBException {
public TethysParameterPacker(TethysControl tethysControl) throws JAXBException {
super();
this.tethysControl = tethysControl;
try {
marshaller = new MarshalXML();
} catch (JAXBException e) {
@ -117,12 +124,43 @@ public class TethysParameterPacker {
if (parameterSet == null) {
return null;
}
// Document document = null;
// try {
// document = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
// } catch (ParserConfigurationException e1) {
// e1.printStackTrace();
// get the XML writer ready for a new export ...
xmlWriter.setExcludeDisplaySettings(true);
xmlWriter.makeSettingsList();
/**
* first do the data filter. I can't see any way of doing this
* without creating a doc as was in the helper example.
*/
QName qnamef = new QName(MarshalXML.schema, "datafilter", "ty");
JAXBElement<String> jaxelf = new JAXBElement<String>(
qnamef, String.class, parameterSet.getParentObject().getClass().getCanonicalName());
Document docf = null;
try {
docf = marshaller.marshalToDOM(jaxelf);
} catch (JAXBException | ParserConfigurationException e1) {
e1.printStackTrace();
}
Element elf = docf.getDocumentElement();
elList.add(elf);/**
* Is there a data filter ? If so, write it's
* XML parameters out here.
*/
DataSelector dataSelector = pamDataBlock.getDataSelector(tethysControl.getDataSelectName(), false);
if (dataSelector != null) {
DataSelectParams filterParams = dataSelector.getParams();
if (filterParams != null) {
Element pEl = xmlWriter.writeObjectData(docf, elf, filterParams, null);
// if (pEl != null) {
//// filterEl.appendChild(pEl);
// elf.appendChild(filterEl);
// }
}
}
QName qname = new QName(MarshalXML.schema, "parameters", "ty");
JAXBElement<String> jaxel = new JAXBElement<String>(
qname, String.class, parameterSet.getParentObject().getClass().getCanonicalName());
@ -133,18 +171,14 @@ public class TethysParameterPacker {
e1.printStackTrace();
}
Element el = doc.getDocumentElement();
for (PamParameterData pamParam : parameterSet.getParameterCollection()) {
try {
Object paramData = pamParam.getData();
boolean ok = createElement(doc, el, paramData, pamParam, objectHierarchy);
} catch (IllegalArgumentException | IllegalAccessException e) {
e.printStackTrace();
}
}
elList.add(el);
xmlWriter.setExcludeDisplaySettings(true);
xmlWriter.makeSettingsList();
/**
* Now get the chain of PAMGuard modules for the current detector and for
* all upstream modules.
*/
ArrayList<PamControlledUnit> moduleChain = getParentChain(pamDataBlock);
for (PamControlledUnit pcu : moduleChain) {
if (pcu instanceof PamSettings == false) {

10
src/tethys/swing/DatablockDetectionsPanel.java
View File

@ -20,6 +20,8 @@ import PamView.tables.SwingTableColumnWidths;
import PamguardMVC.PamDataBlock;
import nilus.Detections;
import tethys.TethysControl;
import tethys.TethysState;
import tethys.TethysState.StateType;
import tethys.dbxml.TethysException;
import tethys.detection.StreamDetectionsSummary;
import tethys.niluswraps.PDetections;
@ -75,6 +77,13 @@ public class DatablockDetectionsPanel extends TethysGUIPanel implements StreamTa
tableModel.fireTableDataChanged();
}
@Override
public void updateState(TethysState tethysState) {
if (dataBlock != null) {
selectDataBlock(dataBlock);
}
}
private class MouseActions extends MouseAdapter {
@Override
@ -137,6 +146,7 @@ public class DatablockDetectionsPanel extends TethysGUIPanel implements StreamTa
} catch (TethysException e) {
getTethysControl().showException(e);
}
getTethysControl().exportedDetections(dataBlock);
selectDataBlock(dataBlock); // force table update.
}

2
src/tethys/swing/DatablockSynchPanel.java
View File

@ -28,6 +28,7 @@ import PamguardMVC.PamDataBlock;
import dataMap.OfflineDataMap;
import tethys.TethysControl;
import tethys.TethysState;
import tethys.TethysStateObserver;
import tethys.output.DatablockSynchInfo;
import tethys.species.DataBlockSpeciesManager;
@ -133,6 +134,7 @@ public class DatablockSynchPanel extends TethysGUIPanel {
@Override
public void updateState(TethysState tethysState) {
synchTableModel.fireTableDataChanged();
selectRow();
}
public void addTableObserver(StreamTableObserver observer) {

2
src/tethys/swing/export/AlgorithmCard.java
View File

@ -20,7 +20,7 @@ public class AlgorithmCard extends ExportWizardCard {
private JTextField method, software, version, supportSoftware;
public AlgorithmCard(TethysControl tethysControl, PamDataBlock dataBlock) {
public AlgorithmCard(DetectionsExportWizard detectionsExportWizard, TethysControl tethysControl, PamDataBlock dataBlock) {
super(tethysControl, "Algorithm", dataBlock);
setBorder(new TitledBorder("Algorithm details"));
method = new JTextField(40);

2
src/tethys/swing/export/DescriptionCard.java
View File

@ -10,7 +10,7 @@ public class DescriptionCard extends ExportWizardCard {
private DescriptionTypePanel descriptionPanel;
public DescriptionCard(TethysControl tethysControl, PamDataBlock dataBlock) {
public DescriptionCard(DetectionsExportWizard detectionsExportWizard, TethysControl tethysControl, PamDataBlock dataBlock) {
super(tethysControl, "Description", dataBlock);
this.setLayout(new BorderLayout());
descriptionPanel = new DescriptionTypePanel("Description data", true, true, true);

12
src/tethys/swing/export/DetectionsExportWizard.java
View File

@ -48,10 +48,10 @@ public class DetectionsExportWizard extends PamDialog {
cardPanel = new JPanel(cardLayout);
mainPanel.add(BorderLayout.CENTER, cardPanel);
addCard(algorithmCard = new AlgorithmCard(tethysControl, dataBlock));
addCard(granularityCard = new GranularityCard(tethysControl, dataBlock));
addCard(descriptionCard = new DescriptionCard(tethysControl, dataBlock));
addCard(exportWorkerCard = new ExportWorkerCard(tethysControl, dataBlock));
addCard(algorithmCard = new AlgorithmCard(this, tethysControl, dataBlock));
addCard(granularityCard = new GranularityCard(this, tethysControl, dataBlock));
addCard(descriptionCard = new DescriptionCard(this, tethysControl, dataBlock));
addCard(exportWorkerCard = new ExportWorkerCard(this, tethysControl, dataBlock));
cardLayout.first(cardPanel);
@ -99,6 +99,10 @@ public class DetectionsExportWizard extends PamDialog {
enableControls();
}
public JButton getPreviousButton() {
return prevButton;
}
@Override
public boolean getParams() {
int iCard = getCardIndex();

7
src/tethys/swing/export/ExportWorkerCard.java
View File

@ -36,8 +36,11 @@ public class ExportWorkerCard extends ExportWizardCard implements DetectionExpor
private StreamExportParams streamExportParams;
public ExportWorkerCard(TethysControl tethysControl, PamDataBlock dataBlock) {
private DetectionsExportWizard detectionsExportWizard;
public ExportWorkerCard(DetectionsExportWizard detectionsExportWizard, TethysControl tethysControl, PamDataBlock dataBlock) {
super(tethysControl, "Export", dataBlock);
this.detectionsExportWizard = detectionsExportWizard;
setLayout(new BorderLayout());
setBorder(new TitledBorder("Export data"));
JPanel exPanel = new PamNorthPanel(new GridBagLayout());
@ -146,6 +149,8 @@ public class ExportWorkerCard extends ExportWizardCard implements DetectionExpor
break;
case DetectionExportProgress.STATE_COMPLETE:
progressText.setText("Export complete");
detectionsExportWizard.getCancelButton().setText("Close");
detectionsExportWizard.getPreviousButton().setEnabled(false);
break;
case DetectionExportProgress.STATE_WRITING:
progressText.setText("Writing to Tethys: " + progress.currentDetections.getId());

65
src/tethys/swing/export/GranularityCard.java
View File

@ -37,12 +37,17 @@ public class GranularityCard extends ExportWizardCard {
private JTextArea dataSelectionText;
private JTextField binLength, encounterGap;
private JTextField binLength, minCalls, encounterGap;
private DataSelector dataSelector;
public GranularityCard(TethysControl tethysControl, PamDataBlock dataBlock) {
private DetectionsExportWizard detectionsExportWizard;
private int encounterIndex, binnedIndex;
public GranularityCard(DetectionsExportWizard detectionsExportWizard, TethysControl tethysControl, PamDataBlock dataBlock) {
super(tethysControl, "Granularity", dataBlock);
this.detectionsExportWizard = detectionsExportWizard;
setLayout(new BoxLayout(this, BoxLayout.Y_AXIS));
// granularity
@ -52,29 +57,36 @@ public class GranularityCard extends ExportWizardCard {
GridBagConstraints c = new PamGridBagContraints();
granPanel.setBorder(new TitledBorder("Granularity"));
ButtonGroup granGroup = new ButtonGroup();
GranularityChange gc = new GranularityChange();
for (int i = 0; i < grans.length; i++) {
c.gridx = 0;
granularities[i] = new JRadioButton(PGranularityType.prettyString(grans[i]));
granularities[i].setToolTipText(PGranularityType.toolTip(grans[i]));
granularities[i].addActionListener(gc);
granPanel.add(granularities[i], c);
granGroup.add(granularities[i]);
if (grans[i] == GranularityEnumType.BINNED) {
binnedIndex = i;
c.gridx++;
granPanel.add(new JLabel(" bin duration ", JLabel.RIGHT), c);
c.gridx++;
granPanel.add(binLength = new JTextField(5), c);
c.gridx++;
granPanel.add(new JLabel(" (s) ", JLabel.LEFT), c);
granPanel.add(new JLabel("(s), min Calls", JLabel.LEFT), c);
c.gridx++;
granPanel.add(minCalls = new JTextField(5), c);
binLength.setToolTipText("Time bin duration in seconds");
minCalls.setToolTipText("Minimum number of calls for a bin to be output");
}
if (grans[i] == GranularityEnumType.ENCOUNTER) {
encounterIndex = i;
c.gridx++;
granPanel.add(new JLabel(" min gap ", JLabel.RIGHT), c);
c.gridx++;
granPanel.add(encounterGap = new JTextField(5), c);
c.gridx++;
granPanel.add(new JLabel(" (s) ", JLabel.LEFT), c);
granPanel.add(new JLabel("(s) ", JLabel.LEFT), c);
encounterGap.setToolTipText("Minimum gap between separate encounters");
}
c.gridy++;
}
@ -104,6 +116,21 @@ public class GranularityCard extends ExportWizardCard {
}
private class GranularityChange implements ActionListener {
@Override
public void actionPerformed(ActionEvent e) {
enableControls();
}
}
private void enableControls() {
binLength.setEnabled(granularities[binnedIndex].isSelected());
minCalls.setEnabled(granularities[binnedIndex].isSelected());
encounterGap.setEnabled(granularities[encounterIndex].isSelected());
}
protected void newDataSelection() {
if (dataSelector == null) {
return;
@ -129,6 +156,28 @@ public class GranularityCard extends ExportWizardCard {
break;
}
}
if (streamExportParams.granularity == GranularityEnumType.BINNED) {
try {
streamExportParams.binDurationS = Double.valueOf(binLength.getText());
}
catch (NumberFormatException e) {
return detectionsExportWizard.showWarning("Invalid bin duration parameter");
}
try {
streamExportParams.minBinCount = Integer.valueOf(minCalls.getText());
}
catch (NumberFormatException e) {
return detectionsExportWizard.showWarning("Invalid minimum call count");
}
}
if (streamExportParams.granularity == GranularityEnumType.ENCOUNTER) {
try {
streamExportParams.encounterGapS = Double.valueOf(encounterGap.getText());
}
catch (NumberFormatException e) {
return detectionsExportWizard.showWarning("Invalid encounter gap parameter");
}
}
return streamExportParams.granularity != null;
}
@ -139,7 +188,11 @@ public class GranularityCard extends ExportWizardCard {
for (int i = 0; i < grans.length; i++) {
granularities[i].setSelected(streamExportParams.granularity == grans[i]);
}
binLength.setText(String.format("%3.1f", streamExportParams.binDurationS));
minCalls.setText(String.format("%d", streamExportParams.minBinCount));
encounterGap.setText(String.format("%3.1f", streamExportParams.encounterGapS));
newDataSelection();
enableControls();
}
}