Detections output

Working towards output of Detections ...
This commit is contained in:
Douglas Gillespie 2023-03-15 17:21:35 +00:00
parent 3991a87102
commit f97247f148
9 changed files with 410 additions and 119 deletions

View File

@ -437,7 +437,7 @@ public class PamguardXMLWriter implements PamSettings {
* @param pamSettingsUnit
* @return xml element
*/
private Element writeUnitSettings(Document doc, Element parent, PamSettings pamSettingsUnit) {
public Element writeUnitSettings(Document doc, Element parent, PamSettings pamSettingsUnit) {
int[] settingInds = findSettings(null, pamSettingsUnit.getUnitName());
PamSettings[] settingsObjects = null;
@ -462,7 +462,7 @@ public class PamguardXMLWriter implements PamSettings {
* can be temporary settings objects when writing temporary settings from dialogs.
* @return new XML element.
*/
private Element writeUnitSettings(Document doc, Element parent, PamSettings pamSettingsUnit, PamSettings[] toWrite) {
public Element writeUnitSettings(Document doc, Element parent, PamSettings pamSettingsUnit, PamSettings[] toWrite) {
Element moduleData = doc.createElement("MODULE");
moduleData.setAttribute("Java.class", pamSettingsUnit.getClass().getName());
moduleData.setAttribute("UnitType", pamSettingsUnit.getUnitType());

View File

@ -3055,7 +3055,7 @@ public class PamDataBlock<Tunit extends PamDataUnit> extends PamObservable {
/**
* Gets a data provider for Tethys. These will probably need
* to be bespoke, but for now will autogenerate based on the SALLogging information.
* to be bespoke, but for now will autogenerate based on the SQLLogging information.
* @return the tethysDataProvider
*/
public TethysDataProvider getTethysDataProvider() {

View File

@ -11,6 +11,7 @@ import javax.swing.JMenuItem;
import PamController.PamControlledUnit;
import PamController.PamController;
import PamguardMVC.PamDataBlock;
import tethys.dbxml.DBXMLConnect;
//import nilus.Deployment;
//import nilus.Deployment.Instrument;
import tethys.output.StreamExportParams;
@ -33,9 +34,20 @@ public class TethysControl extends PamControlledUnit {
private TethysExportParams tethysExportParams = new TethysExportParams();
private DBXMLConnect dbxmlConnect;
public TethysControl(String unitName) {
super(unitType, unitName);
dbxmlConnect = new DBXMLConnect(this);
}
/**
* Get DBXML Connector. This class contains all the functions that are needed
* to talk to the database.
* @return DBXML functions.
*/
public DBXMLConnect getDbxmlConnect() {
return dbxmlConnect;
}
@Override

View File

@ -2,6 +2,12 @@ package tethys.dbxml;
import tethys.TethysControl;
/**
* Class containing functions for managing the database connection. Opening, closing,
* writing, keeping track of performance, etc.
* @author Doug Gillespie, Katie O'Laughlin
*
*/
public class DBXMLConnect {
private TethysControl tethysControl;
@ -19,6 +25,6 @@ public class DBXMLConnect {
}
// add whatever calls are necessary to set up schema's etc.
// add whatever calls are necessary ...
}

View File

@ -0,0 +1,34 @@
package tethys.detection;
/**
* Class to help define what the granularity of exported detections
* documents should be. The entire document will be in memory, so it
* may be necessary to add many detections documents into the database
* for a single Deployment.
* @author dg50
*
*/
public class DetectionGranularity {
public enum GRANULARITY {NONE, BINARYFILE, TIME};
/**
* Type of granularity. Are data all in one lump, split by binary file or by time.
*/
GRANULARITY granularity = GRANULARITY.NONE;
/**
* Granularity interval in seconds. Output system will try to round these
* to something with sensible boundaries.
* This field is only needed when using the GRANULARITY.TIME option.
*/
public long granularityIntervalSeconds;
public DetectionGranularity(GRANULARITY granularity, long granularityIntervalSeconds) {
super();
this.granularity = granularity;
this.granularityIntervalSeconds = granularityIntervalSeconds;
}
}

View File

@ -0,0 +1,199 @@
package tethys.detection;
import java.util.ArrayList;
import java.util.List;
import org.w3c.dom.Document;
import PamController.PamControlledUnit;
import PamController.PamSettings;
import PamController.settings.output.xml.PamguardXMLWriter;
import PamguardMVC.PamDataBlock;
import PamguardMVC.PamDataUnit;
import PamguardMVC.dataSelector.DataSelector;
import dataMap.OfflineDataMap;
import dataMap.OfflineDataMapPoint;
import nilus.DataSourceType;
import nilus.Deployment;
import nilus.DetectionEffort;
import nilus.DetectionEffortKind;
import nilus.Detections;
import tethys.TethysControl;
import tethys.TethysTimeFuncs;
import tethys.detection.DetectionGranularity.GRANULARITY;
import tethys.output.StreamExportParams;
import tethys.output.TethysExportParams;
import tethys.pamdata.TethysDataProvider;
import tethys.pamdata.TethysSchema;
public class DetectionsHandler {
private TethysControl tethysControl;
public DetectionsHandler(TethysControl tethysControl) {
super();
this.tethysControl = tethysControl;
}
/**
* Here is where we export data for a specific data stream to Tethys.
*
* @param aDataBlock
* @param aDeployment
* @param tethysExportParams
* @param streamExportParams
*/
public boolean exportDetections(PamDataBlock aDataBlock, Deployment deployment, DetectionGranularity granularity, TethysExportParams tethysExportParams,
StreamExportParams streamExportParams) {
if (granularity == null || granularity.granularity == null) {
granularity = new DetectionGranularity(GRANULARITY.TIME, 3600);
}
switch (granularity.granularity) {
case BINARYFILE:
return exportByBinaryFile(aDataBlock, deployment, tethysExportParams, streamExportParams);
case NONE:
return exportEverything(aDataBlock, deployment, tethysExportParams, streamExportParams);
case TIME:
return exportByTimeChunk(aDataBlock, deployment, granularity.granularityIntervalSeconds, tethysExportParams, streamExportParams);
default:
break;
}
/**
* This will probably need to be passed additional parameters and may also want
* to return something other than void in order to build a bigger Tethys
* document.
*/
/*
* first we'll probably want a reference to the module containing the data. in
* principle this can't get null, since the datablock was found be searching in
* the other direction.
*/
PamControlledUnit pamControlledUnit = aDataBlock.getParentProcess().getPamControlledUnit();
TethysDataProvider dataProvider = aDataBlock.getTethysDataProvider();
Detections detections = new Detections();
detections.setId(deployment.getId());
detections.setDescription(dataProvider.getDescription(deployment, tethysExportParams));
DataSourceType dataSource = new DataSourceType();
dataSource.setDeploymentId(deployment.getId());
// dataSource.setEnsembleId(""); ToDo
detections.setDataSource(dataSource);
detections.setAlgorithm(dataProvider.getAlgorithm());
detections.setUserId("Unknown user");
detections.setEffort(getDetectorEffort(deployment));
return true;
}
private boolean exportByBinaryFile(PamDataBlock dataBlock, Deployment deployment,
TethysExportParams tethysExportParams, StreamExportParams streamExportParams) {
long deploymentStart = TethysTimeFuncs.millisFromGregorianXML(deployment.getDeploymentDetails().getAudioTimeStamp());
long deploymentStop = TethysTimeFuncs.millisFromGregorianXML(deployment.getRecoveryDetails().getAudioTimeStamp());
/*
* there should be a pretty good correspondence between the start of a binary file and the deploymentStart
* since they all derived from the same start clock.
*/
OfflineDataMap dataMap = dataBlock.getPrimaryDataMap();
if (dataMap == null) {
return false;
}
List<OfflineDataMapPoint> mapPoints = dataMap.getMapPoints();
boolean ok = true;
for (OfflineDataMapPoint mapPoint : mapPoints) {
if (mapPoint.getEndTime() < deploymentStart) {
continue;
}
if (mapPoint.getStartTime() >= deploymentStop) {
continue;
}
ok &= loadAndExport(dataBlock, deployment, Math.max(deploymentStart, mapPoint.getStartTime()),
Math.min(deploymentStop, mapPoint.getEndTime()), tethysExportParams, streamExportParams);
}
return ok;
}
private boolean exportEverything(PamDataBlock dataBlock, Deployment deployment,
TethysExportParams tethysExportParams, StreamExportParams streamExportParams) {
long deploymentStart = TethysTimeFuncs.millisFromGregorianXML(deployment.getDeploymentDetails().getAudioTimeStamp());
long deploymentStop = TethysTimeFuncs.millisFromGregorianXML(deployment.getRecoveryDetails().getAudioTimeStamp());
return loadAndExport(dataBlock, deployment, deploymentStart, deploymentStop, tethysExportParams, streamExportParams);
}
private boolean exportByTimeChunk(PamDataBlock dataBlock, Deployment deployment, long granularityIntervalSeconds,
TethysExportParams tethysExportParams, StreamExportParams streamExportParams) {
long deploymentStart = TethysTimeFuncs.millisFromGregorianXML(deployment.getDeploymentDetails().getAudioTimeStamp());
long deploymentStop = TethysTimeFuncs.millisFromGregorianXML(deployment.getRecoveryDetails().getAudioTimeStamp());
long chunkMillis = granularityIntervalSeconds*1000;
long exportStart = deploymentStart / chunkMillis;
exportStart *= chunkMillis;
boolean ok = true;
while (exportStart < deploymentStop) {
ok &= loadAndExport(dataBlock, deployment, Math.max(deploymentStart, exportStart),
Math.min(deploymentStop, exportStart + chunkMillis), tethysExportParams, streamExportParams);
exportStart += chunkMillis;
}
return ok;
}
/**
* Load and export data for a given time period. This may be a complete deployment, it may be a short section. Do as told !
* Hopefully data interval is small enough to hold all in memory - it needs to be if the document will fit in mempory, so should be OK
* @param dataBlock
* @param deployment
* @param max
* @param min
* @param tethysExportParams
* @param streamExportParams
*/
private boolean loadAndExport(PamDataBlock dataBlock, Deployment deployment, long startTimeMillis, long endTimeMillis,
TethysExportParams tethysExportParams, StreamExportParams streamExportParams) {
// load the data
dataBlock.loadViewerData(startTimeMillis, endTimeMillis, null);
DataSelector dataSelector = dataBlock.getDataSelector(tethysControl.getDataSelectName(), false);
/*
* for easier synching, get a copy of the data and also apply the data selector right away so that
* we've a list of exactly the right data.
*/
ArrayList<PamDataUnit> data = dataBlock.getDataCopy(startTimeMillis, endTimeMillis, true, dataSelector);
/*
* Here, make Detection object and add the DetectionEffort data.
*/
for (int i = 0; i < data.size(); i++) {
PamDataUnit dataUnit = data.get(i);
// add many Detecion objects
}
// write to database
return true;
}
// private boolean exportByTimeChunk(PamDataBlock aDataBlock, Deployment deployment, long granularityIntervalSeconds,
// TethysExportParams tethysExportParams, StreamExportParams streamExportParams) {
// // TODO Auto-generated method stub
// return false;
// }
private DetectionEffort getDetectorEffort(Deployment deployment) {
DetectionEffort effort = new DetectionEffort();
effort.setStart(deployment.getDeploymentDetails().getAudioTimeStamp());
effort.setEnd(deployment.getRecoveryDetails().getAudioTimeStamp());
// effort.set // no setter for DetectionEffortKind
List<DetectionEffortKind> effortKinds = effort.getKind();
return effort;
}
}

View File

@ -34,6 +34,9 @@ import tethys.TethysControl;
import tethys.dbxml.DBXMLConnect;
import tethys.deployment.DeploymentHandler;
import tethys.deployment.DeploymentRecoveryPair;
import tethys.detection.DetectionGranularity;
import tethys.detection.DetectionGranularity.GRANULARITY;
import tethys.detection.DetectionsHandler;
import tethys.pamdata.TethysDataProvider;
import tethys.pamdata.TethysSchema;
@ -204,6 +207,7 @@ public class TethysExporter {
return false;
}
ArrayList<Deployment> deploymentDocs = new ArrayList<>();
/*
* This will become the main loop over deployment documents
*/
@ -212,7 +216,7 @@ public class TethysExporter {
Deployment deployment = deploymentHandler.createDeploymentDocument(i++, drd);
// System.out.println(deployment.toString());
deploymentDocs.add(deployment1);
}
@ -222,13 +226,21 @@ public class TethysExporter {
* go through the export params and call something for every data block that's
* enabled.
*/
DetectionsHandler detectionsHandler = new DetectionsHandler(tethysControl);
ArrayList<PamDataBlock> allDataBlocks = PamController.getInstance().getDataBlocks();
for (PamDataBlock aDataBlock : allDataBlocks) {
StreamExportParams streamExportParams = tethysExportParams.getStreamParams(aDataBlock);
if (streamExportParams == null || !streamExportParams.selected) {
continue; // not interested in this one.
/**
* Outer loop is through deployemnt documents. Will then export detections within each
* deployment detector by detector
*/
for (Deployment aDeployment : deploymentDocs) {
for (PamDataBlock aDataBlock : allDataBlocks) {
StreamExportParams streamExportParams = tethysExportParams.getStreamParams(aDataBlock);
if (streamExportParams == null || !streamExportParams.selected) {
continue; // not interested in this one.
}
detectionsHandler.exportDetections(aDataBlock, aDeployment,
new DetectionGranularity(GRANULARITY.TIME, 3600), tethysExportParams, streamExportParams);
}
exportDataStream(aDataBlock, tethysExportParams, streamExportParams);
}
/*
* Then do whatever else is needed to complete the document.
@ -310,112 +322,4 @@ public class TethysExporter {
/**
* Here is where we export data for a specific data stream to Tethys.
*
* @param aDataBlock
* @param tethysExportParams
* @param streamExportParams
*/
private void exportDataStream(PamDataBlock aDataBlock, TethysExportParams tethysExportParams,
StreamExportParams streamExportParams) {
/**
* This will probably need to be passed additional parameters and may also want
* to return something other than void in order to build a bigger Tethys
* document.
*/
/*
* Some examples of how to do whatever is needed to get schema and data out of
* PAMGuard.
*/
/*
* first we'll probably want a reference to the module containing the data. in
* principle this can't get null, since the datablock was found be searching in
* the other direction.
*/
PamControlledUnit pamControlledUnit = aDataBlock.getParentProcess().getPamControlledUnit();
TethysDataProvider dataProvider = aDataBlock.getTethysDataProvider();
PamguardXMLWriter pamXMLWriter = PamguardXMLWriter.getXMLWriter();
if (dataProvider == null) {
return;
}
TethysSchema tethysSchema = dataProvider.getSchema();
/*
* the schema should have a Document object in it. If we wanted to turn that
* into an XML string we can ... (though I'd assume that having the Document is
* more useful)
*/
String schemaXMLString = pamXMLWriter.getAsString(tethysSchema.getXsd(), false);
System.out.printf("Schema for %s is %s\n", aDataBlock.getDataName(), schemaXMLString);
/*
* Get the XML settings for that datablock. This is (or should be the parameters
* that were controlling that module, with adequate data about upstream
* modules). I think this has to go somewhere into the Detections document.
*/
Document doc = pamXMLWriter.writeOneModule((PamSettings) pamControlledUnit, System.currentTimeMillis());
String moduleXML = null;
if (doc != null) {
// this string should be XML of all the settings for the module controlling this
// datablock.
moduleXML = pamXMLWriter.getAsString(doc, true); // change to false to get smaller xml
System.out.printf("Module settings for datablock %s are:\n", moduleXML);
System.out.println(moduleXML);
}
/**
* Now can go through the data. Probably, we'll want to go through all the data
* in the project, but we can hold off on that for now and just go for data that
* are in memory. We'll also have to think a lot about updating parts of the
* database which have been reprocessed - what we want to do, should eventually
* all be options set in the dialog and available within TethysExportParams For
* now though, we're just going to export data that are in memory. Once basic
* export is working, I can easily enough write something which will go through
* an entire data set, go through between two times, etc.
*/
// so this is a way of iterating through the data that are in memory, which will
// do for now ..
// do it with a data copy which can avoid synchronising the entire block for
// what may be a long time
// the copy function is itself synched, and is quite fast, so easier and safe
// this way
ArrayList<PamDataUnit> dataCopy = aDataBlock.getDataCopy();
DataSelector dataSelector = aDataBlock.getDataSelector(tethysControl.getDataSelectName(), false);
int nSkipped = 0;
int nExport = 0;
for (PamDataUnit aData : dataCopy) {
/*
* see if we want this data unit. PAMGuard has a complicated system of data
* selectors specific to each data type. These are centrally managed so you
* don't need to worry too much about them. They are identified by name for each
* data stream and the behaviour here should follow the selections you made in
* the dialog. the data selection system allows different displays to show
* different data, so a stream can have many differently named selectors active
* at any one time, all doing different things in different parts of PAMGuard.
*/
if (dataSelector != null) {
if (dataSelector.scoreData(aData) <= 0) {
nSkipped++;
continue; // don't want this one.
}
}
/*
* then we do whatever we need to do to convert this into something for Tethys
* this might happen in the tethysSchema object for each data stream ????
*/
nExport++;
}
System.out.printf("Exported %d data units and skipped %d in %s", nExport, nSkipped,
aDataBlock.getLongDataName());
}
}

View File

@ -1,11 +1,27 @@
package tethys.pamdata;
import org.w3c.dom.Document;
import java.util.List;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import PamController.PamControlledUnit;
import PamController.PamSettings;
import PamController.PamguardVersionInfo;
import PamController.settings.output.xml.PamguardXMLWriter;
import PamUtils.XMLUtils;
import PamguardMVC.PamDataBlock;
import PamguardMVC.PamDataUnit;
import PamguardMVC.PamProcess;
import generalDatabase.DBSchemaWriter;
import generalDatabase.SQLLogging;
import nilus.AlgorithmType;
import nilus.AlgorithmType.Parameters;
import nilus.Deployment;
import nilus.DescriptionType;
import tethys.output.TethysExportParams;
/**
* Automatically provides Tethys data based on the SQL database interface
@ -16,9 +32,13 @@ import generalDatabase.SQLLogging;
public class AutoTethysProvider implements TethysDataProvider {
private PamDataBlock pamDataBlock;
private PamProcess pamProcess;
private PamControlledUnit pamControlledUnit;
public AutoTethysProvider(PamDataBlock pamDataBlock) {
this.pamDataBlock = pamDataBlock;
pamProcess = pamDataBlock.getParentProcess();
pamControlledUnit = pamProcess.getPamControlledUnit();
}
@Override
@ -39,4 +59,110 @@ public class AutoTethysProvider implements TethysDataProvider {
return null;
}
@Override
public DescriptionType getDescription(Deployment deployment, TethysExportParams tethysExportParams) {
DescriptionType description = new DescriptionType();
String fullUnitName = pamControlledUnit.getUnitType() + " " + pamControlledUnit.getUnitName();
description.setAbstract(fullUnitName);
description.setObjectives(fullUnitName);
description.setMethod(pamControlledUnit.getUnitType());
return description;
}
@Override
public AlgorithmType getAlgorithm() {
AlgorithmType algorithm = new AlgorithmType();
algorithm.setMethod(this.getAlgorithmMethod());
algorithm.setSoftware("PAMGuard");
algorithm.setVersion(PamguardVersionInfo.version);
algorithm.setParameters(this.getAlgorithmParameters());
return algorithm;
}
private Parameters getAlgorithmParameters() {
if (pamControlledUnit instanceof PamSettings == false) {
return null;
}
PamSettings pamSettings = (PamSettings) pamControlledUnit;
Parameters parameters = new Parameters();
List<Element> paramList = parameters.getAny();
Document doc = XMLUtils.createBlankDoc();
PamguardXMLWriter pamXMLWriter = PamguardXMLWriter.getXMLWriter();
Element dummyEl = doc.createElement("MODULES");
doc.appendChild(dummyEl);
PamSettings[] settingsObjs = getSettingsObjects();
if (settingsObjs == null) {
return null;
}
Element settingsEl = pamXMLWriter.writeUnitSettings(doc, dummyEl, pamSettings, settingsObjs);
if (settingsEl == null) {
return null;
}
dummyEl.appendChild(settingsEl);
NodeList childs = settingsEl.getChildNodes();
for (int i = 0; i < childs.getLength(); i++) {
Node el = childs.item(i);
System.out.println(el.getNodeName());
if (el instanceof Element) {
paramList.add((Element) el);
}
}
// Document doc = pamXMLWriter.writeOneModule((PamSettings) pamControlledUnit, System.currentTimeMillis());
// String moduleXML = null;
if (doc != null) {
// this string should be XML of all the settings for the module controlling this
// datablock.
// moduleXML = pamXMLWriter.getAsString(doc, true); // change to false to get smaller xml
// System.out.printf("Module settings for datablock %s are:\n", moduleXML);
// System.out.println(moduleXML);
// Element pamguard = doc.get("PAMGUARD");
// Element modules = (Element) pamguard.getElementsByTagName("MODULES");
// doc.get
// NodeList childs = doc.getChildNodes();
// for (int i = 0; i < childs.getLength(); i++) {
// Node el = childs.item(i);
// System.out.println(el.getNodeName());
// if (el instanceof Element) {
// paramList.add((Element) el);
// }
// }
// String moduleXML = pamXMLWriter.getAsString(doc, true); // change to false to get smaller xml
// System.out.printf("Module settings for datablock %s are:\n%s", this.pamDataBlock.getDataName(), moduleXML);
}
// // try the old say
// Document doc2 = pamXMLWriter.writeOneModule((PamSettings) pamControlledUnit, System.currentTimeMillis());
// String moduleXML = null;
// if (doc2 != null) {
// // this string should be XML of all the settings for the module controlling this
// // datablock.
// moduleXML = pamXMLWriter.getAsString(doc2, true); // change to false to get smaller xml
// System.out.printf("Module settings for datablock %s are:\n%s", pamDataBlock.getDataName(),moduleXML);
// }
//
return parameters;
}
private PamSettings[] getSettingsObjects() {
if (pamControlledUnit instanceof PamSettings) {
PamSettings[] settings = new PamSettings[1];
settings[0] = (PamSettings) pamControlledUnit;
return settings;
}
return null;
}
/**
* Algorithm method. Default is the module name. Can change to a paper citation
* by overriding this
* @return
*/
private String getAlgorithmMethod() {
return pamControlledUnit.getUnitType();
}
}

View File

@ -1,6 +1,10 @@
package tethys.pamdata;
import PamguardMVC.PamDataUnit;
import nilus.AlgorithmType;
import nilus.Deployment;
import nilus.DescriptionType;
import tethys.output.TethysExportParams;
/**
* Any PAMGuard data stream which can provide Detection data to PAMGuard will
@ -29,4 +33,10 @@ public interface TethysDataProvider {
*/
public TethysDataPoint getDataPoint(PamDataUnit pamDataUnit);
public DescriptionType getDescription(Deployment deployment, TethysExportParams tethysExportParams);
public AlgorithmType getAlgorithm();
}