diff --git a/src/PamController/settings/output/xml/PamguardXMLWriter.java b/src/PamController/settings/output/xml/PamguardXMLWriter.java index 37530402..f74edce0 100644 --- a/src/PamController/settings/output/xml/PamguardXMLWriter.java +++ b/src/PamController/settings/output/xml/PamguardXMLWriter.java @@ -581,10 +581,13 @@ public class PamguardXMLWriter implements PamSettings { return el; } - private Element writeObjectData(Document doc, Element el, Object data, ArrayList objectHierarchy) { + public Element writeObjectData(Document doc, Element el, Object data, ArrayList objectHierarchy) { if (data == null) { return null; } + if (objectHierarchy == null) { + objectHierarchy = new ArrayList<>(); + } if (objectHierarchy.contains(data)) { // just write the reference, but nothing else or we'll end up in an infinite loop of objects. Element e = doc.createElement("Object"); diff --git a/src/PamguardMVC/PamDataBlock.java b/src/PamguardMVC/PamDataBlock.java index c3a3ba7b..feff646e 100644 --- a/src/PamguardMVC/PamDataBlock.java +++ b/src/PamguardMVC/PamDataBlock.java @@ -50,6 +50,7 @@ import org.w3c.dom.Element; import Acquisition.AcquisitionControl; import Acquisition.AcquisitionProcess; import pamScrollSystem.ViewLoadObserver; +import tethys.TethysControl; import tethys.pamdata.AutoTethysProvider; import tethys.pamdata.TethysDataProvider; import tethys.species.DataBlockSpeciesManager; @@ -3089,9 +3090,9 @@ public class PamDataBlock extends PamObservable { * to be bespoke, but for now will autogenerate based on the SQLLogging information. * @return the tethysDataProvider */ - public TethysDataProvider getTethysDataProvider() { + public TethysDataProvider getTethysDataProvider(TethysControl tethysControl) { if (tethysDataProvider == null && PamDetection.class.isAssignableFrom(unitClass) && getLogging() != null) { - tethysDataProvider = new AutoTethysProvider(this); + tethysDataProvider = new AutoTethysProvider(tethysControl, this); } return tethysDataProvider; } diff --git a/src/PamguardMVC/dataSelector/DataSelector.java b/src/PamguardMVC/dataSelector/DataSelector.java index a91712b5..b033976c 100644 --- a/src/PamguardMVC/dataSelector/DataSelector.java +++ b/src/PamguardMVC/dataSelector/DataSelector.java @@ -139,7 +139,7 @@ public abstract class DataSelector { } return null; } - + /** * Score a PAMDataUnit. this is used in preference * to a boolean select function so that the user can add different diff --git a/src/tethys/TethysControl.java b/src/tethys/TethysControl.java index 92d44aef..c5f26031 100644 --- a/src/tethys/TethysControl.java +++ b/src/tethys/TethysControl.java @@ -211,7 +211,7 @@ public class TethysControl extends PamControlledUnit implements PamSettings, Tet ArrayList sets = new ArrayList<>(); ArrayList allDataBlocks = PamController.getInstance().getDataBlocks(); for (PamDataBlock aDataBlock : allDataBlocks) { - if (aDataBlock.getTethysDataProvider() != null) { + if (aDataBlock.getTethysDataProvider(this) != null) { sets.add(aDataBlock); } } @@ -335,31 +335,31 @@ public class TethysControl extends PamControlledUnit implements PamSettings, Tet return tethysExportParams; } - /** - * We'll probably want to - * @param parentFrame - */ - protected void tethysExport(JFrame parentFrame) { - TethysExportParams newExportParams = TethysExportDialog.showDialog(parentFrame, this); - if (newExportParams != null) { - // dialog returns null if cancel was pressed. - tethysExportParams = newExportParams; - exportTethysData(tethysExportParams); - } - } - - /** - * We'll arrive here if the dialog has been opened and we want to export Tethys data. - * @param tethysExportParams2 - */ - private void exportTethysData(TethysExportParams tethysExportParams) { - TethysExporter tethysExporter = new TethysExporter(this, tethysExportParams); - tethysExporter.doExport(); - - sendStateUpdate(new TethysState(StateType.TRANSFERDATA)); - countProjectDetections(); - sendStateUpdate(new TethysState(StateType.NEWPAMGUARDSELECTION)); - } +// /** +// * We'll probably want to +// * @param parentFrame +// */ +// protected void tethysExport(JFrame parentFrame) { +// TethysExportParams newExportParams = TethysExportDialog.showDialog(parentFrame, this); +// if (newExportParams != null) { +// // dialog returns null if cancel was pressed. +// tethysExportParams = newExportParams; +// exportTethysData(tethysExportParams); +// } +// } +// +// /** +// * We'll arrive here if the dialog has been opened and we want to export Tethys data. +// * @param tethysExportParams2 +// */ +// private void exportTethysData(TethysExportParams tethysExportParams) { +// TethysExporter tethysExporter = new TethysExporter(this, tethysExportParams); +// tethysExporter.doExport(); +// +// sendStateUpdate(new TethysState(StateType.TRANSFERDATA)); +// countProjectDetections(); +// sendStateUpdate(new TethysState(StateType.NEWPAMGUARDSELECTION)); +// } /** * Get global deployment data. This is a bit of a mess, trying to use a separate module @@ -644,4 +644,14 @@ public class TethysControl extends PamControlledUnit implements PamSettings, Tet return itisFunctions; } + /** + * Called when a detections document has been exported. + * @param dataBlock + */ + public void exportedDetections(PamDataBlock dataBlock) { + sendStateUpdate(new TethysState(StateType.TRANSFERDATA)); + countProjectDetections(); + sendStateUpdate(new TethysState(StateType.NEWPAMGUARDSELECTION)); + } + } diff --git a/src/tethys/dbxml/DBXMLQueries.java b/src/tethys/dbxml/DBXMLQueries.java index a1d07e7f..1275ff86 100644 --- a/src/tethys/dbxml/DBXMLQueries.java +++ b/src/tethys/dbxml/DBXMLQueries.java @@ -541,6 +541,27 @@ public class DBXMLQueries { // String queryBase = "{\"return\":[\"Deployment/Project\"],\"select\":[],\"enclose\":1}"; } + /** + * Find out if a document exists ? + * @param collection + * @param documentId + * @return + */ + public boolean documentExists(String collection, String documentId) { + Queries queries = dbXMLConnect.getTethysQueries(); + String result = null; + try { + result = queries.getDocument(collection, documentId); + } catch (Exception e) { + return false; + } + if (result == null || result.length() == 0) { + return false; + } + + return result.contains(documentId); + } + /** * Count on effort detections in a Detections document * @param docName diff --git a/src/tethys/detection/BinnedGranularityHandler.java b/src/tethys/detection/BinnedGranularityHandler.java new file mode 100644 index 00000000..b39f4550 --- /dev/null +++ b/src/tethys/detection/BinnedGranularityHandler.java @@ -0,0 +1,161 @@ +package tethys.detection; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Set; + +import PamguardMVC.PamDataBlock; +import PamguardMVC.PamDataUnit; +import nilus.Detection; +import nilus.SpeciesIDType; +import tethys.TethysControl; +import tethys.TethysTimeFuncs; +import tethys.output.StreamExportParams; +import tethys.output.TethysExportParams; +import tethys.pamdata.TethysDataProvider; +import tethys.species.DataBlockSpeciesCodes; +import tethys.species.DataBlockSpeciesManager; +import tethys.species.DataBlockSpeciesMap; +import tethys.species.SpeciesMapItem; + +/** + * Binned granularity + * Will have to collect different counts for each type of call for each datablock (if there + * are such things) so a little more complicated than might be expected. + * @author dg50 + * + */ +public class BinnedGranularityHandler extends GranularityHandler { + + private double binDurationSeconds; + + private long binStartMillis, binEndMillis; + + private TethysDataProvider dataProvider; + + private DataBlockSpeciesManager speciesManager; + + private HashMap currentDetections; + + public BinnedGranularityHandler(TethysControl tethysControl, PamDataBlock dataBlock, + TethysExportParams tethysExportParams, StreamExportParams streamExportParams) { + super(tethysControl, dataBlock, tethysExportParams, streamExportParams); + + binDurationSeconds = streamExportParams.binDurationS; + dataProvider = dataBlock.getTethysDataProvider(tethysControl); + speciesManager = dataBlock.getDatablockSpeciesManager(); + + currentDetections = new HashMap(); + } + + @Override + public void prepare(long timeMillis) { + long binStart = DetectionsHandler.roundDownBinStart(timeMillis, (long) (binDurationSeconds*1000)); + startBin(binStart); + } + + private void startBin(long timeMillis) { + binStartMillis = timeMillis; + binEndMillis = binStartMillis + (long) (binDurationSeconds*1000.); + /* + * now make a Detection object for every possible species that + * this might throw out. + */ + ArrayList speciesCodes = speciesManager.getAllSpeciesCodes(); + String defaultCode = speciesManager.getDefaultSpeciesCode(); + Detection det; + currentDetections.put(defaultCode, det = new Detection()); + det.setStart(TethysTimeFuncs.xmlGregCalFromMillis(binStartMillis)); + det.setEnd(TethysTimeFuncs.xmlGregCalFromMillis(binEndMillis)); + det.setCount(BigInteger.ZERO); + det.setChannel(BigInteger.ZERO); + // add codes at end, just before output. + if (speciesCodes != null) { + for (String code : speciesCodes) { + currentDetections.put(code, det = new Detection()); + det.setStart(TethysTimeFuncs.xmlGregCalFromMillis(binStartMillis)); + det.setEnd(TethysTimeFuncs.xmlGregCalFromMillis(binEndMillis)); + det.setCount(BigInteger.ZERO); + det.setChannel(BigInteger.ZERO); + } + } + } + + @Override + public Detection[] addDataUnit(PamDataUnit dataUnit) { + Detection[] detections = null; + if (dataUnit.getTimeMilliseconds() >= binEndMillis) { + detections = closeBins(dataUnit.getTimeMilliseconds()); + } + String speciesCode = speciesManager.getSpeciesCode(dataUnit); + Detection det = currentDetections.get(speciesCode); + if (det != null) { + /* + * Increase the detection count + */ + int count = det.getCount().intValue(); + count++; + det.setCount(BigInteger.valueOf(count)); + /* + * Add to the channel map too ... + */ + int channel = det.getChannel().intValue(); + channel |= dataUnit.getChannelBitmap(); + det.setChannel(BigInteger.valueOf(channel)); + } + return detections; + } + + /** + * Called when units arrive after end of current bin, and also + * at end of deployment output, to get that last bine. + * @param timeMilliseconds + * @return + */ + private Detection[] closeBins(long timeMilliseconds) { + Set speciesKeys = currentDetections.keySet(); + int n = speciesKeys.size(); + int nGood = 0; + DataBlockSpeciesMap speciesMap = speciesManager.getDatablockSpeciesMap(); + Detection detections[] = new Detection[n]; + for (String key : speciesKeys) { + Detection det = currentDetections.get(key); + int callCount = det.getCount().intValue(); + if (callCount < Math.max(streamExportParams.minBinCount,1)) { + continue; + } + SpeciesMapItem speciesStuff = speciesMap.getItem(key); // should be non null! + if (speciesStuff == null) { + continue; + } + SpeciesIDType species = new SpeciesIDType(); + species.setValue(BigInteger.valueOf(speciesStuff.getItisCode())); + det.setSpeciesId(species); + if (speciesStuff.getCallType() != null) { + det.getCall().add(speciesStuff.getCallType()); + } + detections[nGood++] = det; + } + + + // finally, start new bins (not really needed on last call, but do anyway). + startBin(binEndMillis); + + /* + * Clean up the end of the array and return detections that have enough calls. + */ + if (nGood == 0) { + return null; + } + detections = Arrays.copyOf(detections, nGood); + return detections; + } + + @Override + public Detection[] cleanup(long timeMillis) { + return closeBins(timeMillis); + } + +} diff --git a/src/tethys/detection/CallGranularityHandler.java b/src/tethys/detection/CallGranularityHandler.java new file mode 100644 index 00000000..4ff9a888 --- /dev/null +++ b/src/tethys/detection/CallGranularityHandler.java @@ -0,0 +1,40 @@ +package tethys.detection; + +import PamguardMVC.PamDataBlock; +import PamguardMVC.PamDataUnit; +import nilus.Detection; +import tethys.TethysControl; +import tethys.output.StreamExportParams; +import tethys.output.TethysExportParams; +import tethys.pamdata.TethysDataProvider; + +public class CallGranularityHandler extends GranularityHandler { + + private TethysDataProvider dataProvider; + + public CallGranularityHandler(TethysControl tethysControl, PamDataBlock dataBlock, + TethysExportParams tethysExportParams, StreamExportParams streamExportParams) { + super(tethysControl, dataBlock, tethysExportParams, streamExportParams); + + dataProvider = dataBlock.getTethysDataProvider(tethysControl); + + } + + @Override + public void prepare(long timeMillis) { + // never anything to do here for call level granularity. + } + + @Override + public Detection[] addDataUnit(PamDataUnit dataUnit) { + Detection det = dataProvider.createDetection(dataUnit, tethysExportParams, streamExportParams); + return toDetectionArray(det); + } + + @Override + public Detection[] cleanup(long timeMillis) { + // never anything to do here for call level granularity. + return null; + } + +} diff --git a/src/tethys/detection/DetectionGranularity.java b/src/tethys/detection/DetectionGranularity.java deleted file mode 100644 index 7380fb67..00000000 --- a/src/tethys/detection/DetectionGranularity.java +++ /dev/null @@ -1,34 +0,0 @@ -package tethys.detection; - -/** - * Class to help define what the granularity of exported detections - * documents should be. The entire document will be in memory, so it - * may be necessary to add many detections documents into the database - * for a single Deployment. - * @author dg50 - * - */ -public class DetectionGranularity { - - public enum GRANULARITY {NONE, BINARYFILE, TIME}; - - /** - * Type of granularity. Are data all in one lump, split by binary file or by time. - */ - GRANULARITY granularity = GRANULARITY.NONE; - - /** - * Granularity interval in seconds. Output system will try to round these - * to something with sensible boundaries. - * This field is only needed when using the GRANULARITY.TIME option. - */ - public long granularityIntervalSeconds; - - public DetectionGranularity(GRANULARITY granularity, long granularityIntervalSeconds) { - super(); - this.granularity = granularity; - this.granularityIntervalSeconds = granularityIntervalSeconds; - } - - -} diff --git a/src/tethys/detection/DetectionsHandler.java b/src/tethys/detection/DetectionsHandler.java index bc5992db..bd1c8370 100644 --- a/src/tethys/detection/DetectionsHandler.java +++ b/src/tethys/detection/DetectionsHandler.java @@ -1,33 +1,20 @@ package tethys.detection; -import java.math.BigInteger; import java.util.ArrayList; import java.util.List; import javax.swing.SwingWorker; -import javax.xml.XMLConstants; -import javax.xml.bind.JAXBException; -import javax.xml.parsers.ParserConfigurationException; - -import org.pamguard.x3.sud.SUDClickDetectorInfo; -import org.w3c.dom.Document; -import org.w3c.dom.Element; import PamController.PamControlledUnit; import PamController.PamguardVersionInfo; -import PamController.settings.output.xml.PamguardXMLWriter; import PamModel.PamPluginInterface; -import PamUtils.PamCalendar; -import PamUtils.XMLUtils; import PamguardMVC.PamDataBlock; import PamguardMVC.PamDataUnit; import PamguardMVC.PamProcess; import PamguardMVC.dataSelector.DataSelector; import dataMap.OfflineDataMap; import dataMap.OfflineDataMapPoint; -import metadata.deployment.DeploymentData; import nilus.AlgorithmType; -import nilus.AlgorithmType.Parameters; import nilus.AlgorithmType.SupportSoftware; import nilus.DataSourceType; import nilus.Deployment; @@ -38,43 +25,38 @@ import nilus.DetectionGroup; import nilus.Detections; import nilus.Helper; import tethys.TethysControl; -import tethys.TethysState; -import tethys.TethysState.StateType; -import tethys.deployment.DeploymentHandler; -import tethys.TethysStateObserver; import tethys.TethysTimeFuncs; import tethys.dbxml.DBXMLConnect; import tethys.dbxml.TethysException; -import tethys.detection.DetectionGranularity.GRANULARITY; +import tethys.deployment.DeploymentHandler; import tethys.niluswraps.PDeployment; import tethys.niluswraps.PDetections; +import tethys.niluswraps.TethysCollections; import tethys.output.StreamExportParams; import tethys.output.TethysExportParams; -import tethys.pamdata.TethysDataPoint; import tethys.pamdata.TethysDataProvider; -import tethys.swing.export.ExportWorkerCard; public class DetectionsHandler { private TethysControl tethysControl; - + public int uniqueDetectionsId=1; public int uniqueDetectionId; private volatile boolean activeExport; private ExportWorker exportWorker; - + public DetectionsHandler(TethysControl tethysControl) { super(); this.tethysControl = tethysControl; } - - + + /** - * Get a list of Detections documents associated with a particular data stream for - * this data set (not the entire project). + * Get a list of Detections documents associated with a particular data stream for + * this data set (not the entire project). * @param dataBlock */ public StreamDetectionsSummary getStreamDetections(PamDataBlock dataBlock) { @@ -84,21 +66,21 @@ public class DetectionsHandler { /** * Get a list of Detections documents associated with a particular data block for the list of deployments - * documents. Group them by abstract or something + * documents. Group them by abstract or something * @param dataBlock * @param deployments * @return */ public StreamDetectionsSummary getStreamDetections(PamDataBlock dataBlock, ArrayList deployments) { - // get the basic data for each document including it's Description. - + // get the basic data for each document including it's Description. + ArrayList detectionsDocs = new ArrayList<>(); for (PDeployment aDep : deployments) { ArrayList someNames = tethysControl.getDbxmlQueries().getDetectionsDocuments(dataBlock, aDep.deployment.getId()); if (someNames == null) { continue; } - // no have a list of all the Detections documents of interest for this datablock. + // no have a list of all the Detections documents of interest for this datablock. for (String aDoc : someNames) { Detections detections = tethysControl.getDbxmlQueries().getDetectionsDocInfo(aDoc); int count = tethysControl.getDbxmlQueries().countDetections2(aDoc); @@ -114,7 +96,7 @@ public class DetectionsHandler { // * Here is where we export data for a specific data stream to Tethys. // * // * @param aDataBlock -// * @param aDeployment +// * @param aDeployment // * @param tethysExportParams // * @param streamExportParams // */ @@ -135,7 +117,7 @@ public class DetectionsHandler { // } // // return false; -// +// // // } // @@ -145,7 +127,7 @@ public class DetectionsHandler { // long deploymentStop = TethysTimeFuncs.millisFromGregorianXML(deployment.getRecoveryDetails().getAudioTimeStamp()); // /* // * there should be a pretty good correspondence between the start of a binary file and the deploymentStart -// * since they all derived from the same start clock. +// * since they all derived from the same start clock. // */ // OfflineDataMap dataMap = dataBlock.getPrimaryDataMap(); // if (dataMap == null) { @@ -160,11 +142,11 @@ public class DetectionsHandler { // if (mapPoint.getStartTime() >= deploymentStop) { // continue; // } -// ok &= loadAndExport(dataBlock, deployment, Math.max(deploymentStart, mapPoint.getStartTime()), +// ok &= loadAndExport(dataBlock, deployment, Math.max(deploymentStart, mapPoint.getStartTime()), // Math.min(deploymentStop, mapPoint.getEndTime()), tethysExportParams, streamExportParams); // } -// -// +// +// // return ok; // } // @@ -185,11 +167,11 @@ public class DetectionsHandler { // exportStart *= chunkMillis; // boolean ok = true; // while (exportStart < deploymentStop) { -// ok &= loadAndExport(dataBlock, deployment, Math.max(deploymentStart, exportStart), +// ok &= loadAndExport(dataBlock, deployment, Math.max(deploymentStart, exportStart), // Math.min(deploymentStop, exportStart + chunkMillis), tethysExportParams, streamExportParams); // exportStart += chunkMillis; // } -// +// // return ok; // } // @@ -210,11 +192,11 @@ public class DetectionsHandler { // DataSelector dataSelector = dataBlock.getDataSelector(tethysControl.getDataSelectName(), false); // /* // * for easier synching, get a copy of the data and also apply the data selector right away so that -// * we've a list of exactly the right data. +// * we've a list of exactly the right data. // */ // ArrayList data = dataBlock.getDataCopy(startTimeMillis, endTimeMillis, true, dataSelector); // /* -// * Here, make Detection object and add the DetectionEffort data. +// * Here, make Detection object and add the DetectionEffort data. // */ // DeploymentData globalDeplData = tethysControl.getGlobalDeplopymentData(); // TethysDataProvider dataProvider = dataBlock.getTethysDataProvider(); @@ -235,25 +217,25 @@ public class DetectionsHandler { // List detectionList = detectionGroup.getDetection(); // for (int i = 0; i < data.size(); i++) { // PamDataUnit dataUnit = data.get(i); -// Detection detection = dataProvider.createDetection(dataUnit, tethysExportParams, streamExportParams); +// Detection detection = dataProvider.createDetection(dataUnit, tethysExportParams, streamExportParams); // if (detection != null) { // detectionList.add(detection); // } // } -// System.out.printf("Exporting %d %s detections for time period %s to %s\n", detectionList.size(), dataBlock.getDataName(), +// System.out.printf("Exporting %d %s detections for time period %s to %s\n", detectionList.size(), dataBlock.getDataName(), // detections.getEffort().getStart().toString(), detections.getEffort().getEnd().toString()); // /* // * We should now have a fully populated Detections object, so write it to the database -// * using functions in DBXMLConnect +// * using functions in DBXMLConnect // */ // ArrayList detectionDocuments = new ArrayList(); // detectionDocuments.add(detections); -// -//// tethysControl.getDbxmlConnect().postToTethys(detectionDocuments); // call whatever you need to call in here to write the Detections. -// -// +// +//// tethysControl.getDbxmlConnect().postToTethys(detectionDocuments); // call whatever you need to call in here to write the Detections. +// +// // return true; -// +// // } // private boolean exportByTimeChunk(PamDataBlock aDataBlock, Deployment deployment, long granularityIntervalSeconds, @@ -272,15 +254,15 @@ public class DetectionsHandler { // effort.set // no setter for DetectionEffortKind List effortKinds = effort.getKind(); - TethysDataProvider dataProvider = dataBlock.getTethysDataProvider(); + TethysDataProvider dataProvider = dataBlock.getTethysDataProvider(tethysControl); dataProvider.getEffortKinds(pDeployment, effortKinds, exportParams); - - + + return effort; } /** - * Method string for Detections Algorithm documents. + * Method string for Detections Algorithm documents. * @param dataBlock * @return */ @@ -290,11 +272,11 @@ public class DetectionsHandler { } PamProcess process = dataBlock.getParentProcess(); return "PAMGuard " + process.getProcessName(); - + } /** - * Software string for Detections Algorithm documents. + * Software string for Detections Algorithm documents. * @param dataBlock * @return */ @@ -306,7 +288,7 @@ public class DetectionsHandler { } /** - * Software string for Detections Algorithm documents. + * Software string for Detections Algorithm documents. * @param dataBlock * @return */ @@ -324,31 +306,31 @@ public class DetectionsHandler { return plugin.getVersion(); } } - + public String getSupportSoftware(PamDataBlock dataBlock) { return "PAMGuard"; } - + public String getSupportSoftwareVersion(PamDataBlock dataBlock) { -// should try to dig into the binary store and get the version from there. +// should try to dig into the binary store and get the version from there. return PamguardVersionInfo.version; } // /** -// * Get a prefix for a id for a Detections document. This is just the project name +// * Get a prefix for a id for a Detections document. This is just the project name // * and the datablock name. Something may need to be added to allow for multiple -// * analysis going into one database. +// * analysis going into one database. // * @param project // * @param dataBlock -// * @return Detections document prefix. +// * @return Detections document prefix. // */ // public static final String getDetectionsDocIdPrefix(String project, PamDataBlock dataBlock) { // return project + "_" + dataBlock.getDataName(); // } /** - * Detections will be exported in a separate worker thread since export may take some time and - * the user should be given ample opportunity to cancel it. - * @param pamDataBlock + * Detections will be exported in a separate worker thread since export may take some time and + * the user should be given ample opportunity to cancel it. + * @param pamDataBlock * @param streamExportParams * @param exportWorkerCard */ @@ -358,13 +340,25 @@ public class DetectionsHandler { exportWorker = new ExportWorker(pamDataBlock, streamExportParams, exportObserver); exportWorker.execute(); } - + public void cancelExport() { activeExport = false; } /** - * Export detections in all deployments for this PAMGuard dataset. + * Round a bin start so that it's aligned correctly with + * day starts. + * @param binStart + * @param binInterval + * @return + */ + public static long roundDownBinStart(long binStart, long binInterval) { + binStart/=binInterval; + return binStart*binInterval; + } + + /** + * Export detections in all deployments for this PAMGuard dataset. * @param dataBlock * @param streamExportParams * @param exportObserver @@ -372,8 +366,8 @@ public class DetectionsHandler { */ private int exportDetections(PamDataBlock dataBlock, StreamExportParams streamExportParams, DetectionExportObserver exportObserver) { /* - * This is currently called for the entire dataset, but we will need to loop over specific Deployment documents - * and export the content of each separately. + * This is currently called for the entire dataset, but we will need to loop over specific Deployment documents + * and export the content of each separately. */ TethysExportParams exportParams = tethysControl.getTethysExportParams(); DBXMLConnect dbxmlConnect = tethysControl.getDbxmlConnect(); @@ -381,28 +375,29 @@ public class DetectionsHandler { ArrayList deployments = depHandler.getMatchedDeployments(); Detections currentDetections = null; OfflineDataMap dataMap = dataBlock.getPrimaryDataMap(); - TethysDataProvider dataProvider = dataBlock.getTethysDataProvider(); DataSelector dataSelector = dataBlock.getDataSelector(tethysControl.getDataSelectName(), false); int totalCount = dataMap.getDataCount(); int skipCount = 0; int exportCount = 0; long lastUnitTime = 0; DetectionExportProgress prog; + GranularityHandler granularityHandler = GranularityHandler.getHandler(streamExportParams.granularity, tethysControl, dataBlock, exportParams, streamExportParams); for (PDeployment deployment : deployments) { int documentCount = 0; - prog = new DetectionExportProgress(deployment, null, + prog = new DetectionExportProgress(deployment, null, lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_GATHERING); exportObserver.update(prog); - // export everything in that deployment. - // need to loop through all map points in this interval. + granularityHandler.prepare(deployment.getAudioStart()); + // export everything in that deployment. + // need to loop through all map points in this interval. List mapPoints = dataMap.getMapPoints(); for (OfflineDataMapPoint mapPoint : mapPoints) { - if (activeExport == false) { - prog = new DetectionExportProgress(deployment, currentDetections, + if (!activeExport) { + prog = new DetectionExportProgress(deployment, currentDetections, lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_CANCELED); exportObserver.update(prog); } - + if (currentDetections == null) { currentDetections = startDetectionsDocument(deployment, dataBlock, streamExportParams); currentDetections.getEffort().setStart(TethysTimeFuncs.xmlGregCalFromMillis(mapPoint.getStartTime())); @@ -418,19 +413,30 @@ public class DetectionsHandler { skipCount += dataBlock.getUnitsCount() - dataCopy.size(); DetectionGroup onEffort = currentDetections.getOnEffort(); for (PamDataUnit dataUnit : dataCopy) { - Detection det = dataProvider.createDetection(dataUnit, exportParams, streamExportParams); - exportCount++; - documentCount++; - onEffort.getDetection().add(det); + /* + * Here is where we need to handle the different granularities. + */ + Detection dets[] = granularityHandler.addDataUnit(dataUnit); + if (dets != null) { + for (int dd = 0; dd < dets.length; dd++) { + exportCount++; + documentCount++; + onEffort.getDetection().add(dets[dd]); + } + } +// Detection det = dataProvider.createDetection(dataUnit, exportParams, streamExportParams); +// exportCount++; +// documentCount++; +// onEffort.getDetection().add(det); lastUnitTime = dataUnit.getTimeMilliseconds(); } - - prog = new DetectionExportProgress(deployment, currentDetections, + + prog = new DetectionExportProgress(deployment, currentDetections, lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_GATHERING); exportObserver.update(prog); - + if (documentCount > 500000 && mapPoint != dataMap.getLastMapPoint()) { - prog = new DetectionExportProgress(deployment, currentDetections, + prog = new DetectionExportProgress(deployment, currentDetections, lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_WRITING); exportObserver.update(prog); closeDetectionsDocument(currentDetections, mapPoint.getEndTime()); @@ -443,8 +449,17 @@ public class DetectionsHandler { } } + if (currentDetections != null) { - prog = new DetectionExportProgress(deployment, currentDetections, + Detection dets[] = granularityHandler.cleanup(deployment.getAudioEnd()); + if (dets != null) { + for (int dd = 0; dd < dets.length; dd++) { + exportCount++; + documentCount++; + currentDetections.getOnEffort().getDetection().add(dets[dd]); + } + } + prog = new DetectionExportProgress(deployment, currentDetections, lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_WRITING); closeDetectionsDocument(currentDetections, deployment.getAudioEnd()); try { @@ -456,7 +471,7 @@ public class DetectionsHandler { } } - prog = new DetectionExportProgress(null, null, + prog = new DetectionExportProgress(null, null, lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_COMPLETE); exportObserver.update(prog); return DetectionExportProgress.STATE_COMPLETE; @@ -472,7 +487,17 @@ public class DetectionsHandler { } String prefix = deployment.deployment.getId(); - detections.setId(String.format("%s_%d", prefix, uniqueDetectionsId++)); + String fullId = ""; + /* + * Check the document name isn't already used and increment id as necessary. + */ + while (true) { + fullId = String.format("%s_%d", prefix, uniqueDetectionsId++); + if (!tethysControl.getDbxmlQueries().documentExists(TethysCollections.Detections.toString(), fullId)) { + break; + } + } + detections.setId(fullId); // detections.setDescription(dataProvider.getDescription(deployment, tethysExportParams)); detections.setDescription(exportParams.getNilusDetectionDescription()); DataSourceType dataSource = new DataSourceType(); @@ -480,8 +505,8 @@ public class DetectionsHandler { // dataSource.setEnsembleId(""); ToDo detections.setDataSource(dataSource); AlgorithmType algorithm = detections.getAlgorithm(); - - TethysDataProvider dataProvider = dataBlock.getTethysDataProvider(); + + TethysDataProvider dataProvider = dataBlock.getTethysDataProvider(tethysControl); if (dataProvider != null) { algorithm = dataProvider.getAlgorithm(); // detections.setAlgorithm(algorithm); @@ -489,7 +514,7 @@ public class DetectionsHandler { algorithm.setMethod(getMethodString(dataBlock)); algorithm.setSoftware(getSoftwareString(dataBlock)); algorithm.setVersion(getVersionString(dataBlock)); - + List supSoft = algorithm.getSupportSoftware(); SupportSoftware supportSoft = new SupportSoftware(); supportSoft.setSoftware(getSupportSoftware(dataBlock)); @@ -498,13 +523,13 @@ public class DetectionsHandler { detections.setAlgorithm(algorithm); detections.setUserId("Unknown user"); detections.setEffort(getDetectorEffort(deployment, dataBlock, exportParams)); - + return detections; } /** * Close a detections document. This basically just means rewriting the end time and it's only - * important in the event that a document got too big and has to be restarted. + * important in the event that a document got too big and has to be restarted. * @param detections * @param audioEnd */ @@ -517,7 +542,7 @@ public class DetectionsHandler { private PamDataBlock dataBlock; private StreamExportParams exportParams; private DetectionExportObserver exportObserver; - + public ExportWorker(PamDataBlock dataBlock, StreamExportParams exportParams, DetectionExportObserver exportObserver) { super(); @@ -525,21 +550,28 @@ public class DetectionsHandler { this.exportParams = exportParams; this.exportObserver = exportObserver; } - + public void publish(DetectionExportProgress exportProgress) { super.publish(exportProgress); } - + @Override protected Integer doInBackground() throws Exception { - // eventually need to switch over the four granularity options here. - return exportDetections(dataBlock, exportParams, this); + Integer ans = null; + try { + ans = exportDetections(dataBlock, exportParams, this); + } + catch (Exception e) { + e.printStackTrace(); + } + return ans; } @Override protected void done() { // this. DetectionExportProgress prog = new DetectionExportProgress(null, null, 0, 0, 0, 0, DetectionExportProgress.STATE_COMPLETE); + tethysControl.exportedDetections(dataBlock); exportObserver.update(prog); } @@ -554,6 +586,6 @@ public class DetectionsHandler { public void update(DetectionExportProgress progress) { publish(progress); } - + } } diff --git a/src/tethys/detection/EncounterGranularityHandler.java b/src/tethys/detection/EncounterGranularityHandler.java new file mode 100644 index 00000000..061e018e --- /dev/null +++ b/src/tethys/detection/EncounterGranularityHandler.java @@ -0,0 +1,36 @@ +package tethys.detection; + +import PamguardMVC.PamDataBlock; +import PamguardMVC.PamDataUnit; +import nilus.Detection; +import tethys.TethysControl; +import tethys.output.StreamExportParams; +import tethys.output.TethysExportParams; + +public class EncounterGranularityHandler extends GranularityHandler { + + public EncounterGranularityHandler(TethysControl tethysControl, PamDataBlock dataBlock, + TethysExportParams tethysExportParams, StreamExportParams streamExportParams) { + super(tethysControl, dataBlock, tethysExportParams, streamExportParams); + // TODO Auto-generated constructor stub + } + + @Override + public void prepare(long timeMillis) { + // TODO Auto-generated method stub + + } + + @Override + public Detection[] addDataUnit(PamDataUnit dataUnit) { + // TODO Auto-generated method stub + return null; + } + + @Override + public Detection[] cleanup(long timeMillis) { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/src/tethys/detection/GranularityHandler.java b/src/tethys/detection/GranularityHandler.java new file mode 100644 index 00000000..e4e11da2 --- /dev/null +++ b/src/tethys/detection/GranularityHandler.java @@ -0,0 +1,100 @@ +package tethys.detection; + +import PamguardMVC.PamDataBlock; +import PamguardMVC.PamDataUnit; +import nilus.Detection; +import nilus.GranularityEnumType; +import tethys.TethysControl; +import tethys.output.StreamExportParams; +import tethys.output.TethysExportParams; + +public abstract class GranularityHandler { + + protected TethysControl tethysControl; + + protected PamDataBlock dataBlock; + + protected TethysExportParams tethysExportParams; + + protected StreamExportParams streamExportParams; + + /** + * @param tethysControl + * @param dataBlock + * @param tethysExportParams + * @param streamExportParams + */ + public GranularityHandler(TethysControl tethysControl, PamDataBlock dataBlock, + TethysExportParams tethysExportParams, StreamExportParams streamExportParams) { + this.tethysControl = tethysControl; + this.dataBlock = dataBlock; + this.tethysExportParams = tethysExportParams; + this.streamExportParams = streamExportParams; + } + + /** + * Prepare to start, passing the start time of the effort + * or of the first time bin for binned granularity types. + * @param timeMillis + */ + public abstract void prepare(long timeMillis); + + /** + * Put a data unit into a Detection object. for Call granularity + * this will probably return every time. For binned and encounter + * types this will only return at the end of a bin / encounter + * @param dataUnit + * @return Detection object, but only when ready to be added to Detections + */ + public abstract Detection[] addDataUnit(PamDataUnit dataUnit); + + /** + * Called after end end of all data units to get the last bin / encounter.

+ * + * @param timeMillis end time of effort or last bin in milliseconds. + * @return null for Call granularity, otherwise may be non null for binned or encounter. + */ + public abstract Detection[] cleanup(long timeMillis); + + /** + * Convert a single detection to a one element array since that's what' + * most functions need to return. + * @param det + * @return + */ + protected Detection[] toDetectionArray(Detection det) { + if (det == null) { + return null; + } + Detection[] dets = new Detection[1]; + dets[0] = det; + return dets; + } + + /** + * Create the correct type of granularity handler to put individual data units into + * Detection objects. + * @param granularity + * @param tethysControl + * @param dataBlock + * @param tethysExportParams + * @param streamExportParams + * @return + */ + public static GranularityHandler getHandler(GranularityEnumType granularity, TethysControl tethysControl, PamDataBlock dataBlock, + TethysExportParams tethysExportParams, StreamExportParams streamExportParams) { + switch (granularity) { + case BINNED: + return new BinnedGranularityHandler(tethysControl, dataBlock, tethysExportParams, streamExportParams); + case CALL: + return new CallGranularityHandler(tethysControl, dataBlock, tethysExportParams, streamExportParams); + case ENCOUNTER: + new EncounterGranularityHandler(tethysControl, dataBlock, tethysExportParams, streamExportParams); + case GROUPED: + return new GroupedGranularityHandler(tethysControl, dataBlock, tethysExportParams, streamExportParams); + default: + break; + } + return null; + } +} diff --git a/src/tethys/detection/GroupedGranularityHandler.java b/src/tethys/detection/GroupedGranularityHandler.java new file mode 100644 index 00000000..298913a3 --- /dev/null +++ b/src/tethys/detection/GroupedGranularityHandler.java @@ -0,0 +1,36 @@ +package tethys.detection; + +import PamguardMVC.PamDataBlock; +import PamguardMVC.PamDataUnit; +import nilus.Detection; +import tethys.TethysControl; +import tethys.output.StreamExportParams; +import tethys.output.TethysExportParams; + +public class GroupedGranularityHandler extends GranularityHandler { + + public GroupedGranularityHandler(TethysControl tethysControl, PamDataBlock dataBlock, + TethysExportParams tethysExportParams, StreamExportParams streamExportParams) { + super(tethysControl, dataBlock, tethysExportParams, streamExportParams); + // TODO Auto-generated constructor stub + } + + @Override + public void prepare(long timeMillis) { + // TODO Auto-generated method stub + + } + + @Override + public Detection[] addDataUnit(PamDataUnit dataUnit) { + // TODO Auto-generated method stub + return null; + } + + @Override + public Detection[] cleanup(long timeMillis) { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/src/tethys/output/DatablockSynchInfo.java b/src/tethys/output/DatablockSynchInfo.java index 7664637b..d58be25c 100644 --- a/src/tethys/output/DatablockSynchInfo.java +++ b/src/tethys/output/DatablockSynchInfo.java @@ -8,7 +8,7 @@ import tethys.TethysControl; * All the information needed to populate a table row in the synchronisation table. * some will need to be set as rarely as possible since it may * be slow to update.
- * This needs to sit alongside the StreamExportParams objects since those others are serialisable wheras + * This needs to sit alongside the StreamExportParams objects since those others are serialisable whereas * there is a lot of stuff in here which isn't. * @author dg50 * diff --git a/src/tethys/output/StreamExportParams.java b/src/tethys/output/StreamExportParams.java index 2db881fa..27e2c2a6 100644 --- a/src/tethys/output/StreamExportParams.java +++ b/src/tethys/output/StreamExportParams.java @@ -31,6 +31,12 @@ public class StreamExportParams implements Serializable { public GranularityEnumType granularity = GranularityEnumType.CALL; + public double binDurationS = 60; + + public double encounterGapS = 60; + + public int minBinCount = 1; + /* * Can't have this here since it isn't serializable. */ diff --git a/src/tethys/output/TethysExporter.java b/src/tethys/output/TethysExporter.java index 96e8c38c..a3cc0def 100644 --- a/src/tethys/output/TethysExporter.java +++ b/src/tethys/output/TethysExporter.java @@ -1,32 +1,16 @@ package tethys.output; -import java.io.IOException; -import java.io.PrintWriter; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Marshaller; - -import org.w3c.dom.Document; - import Acquisition.AcquisitionControl; import Acquisition.AcquisitionProcess; import Array.ArrayManager; import Array.Hydrophone; import Array.PamArray; import Array.SnapshotGeometry; -import PamController.PamControlledUnit; import PamController.PamController; -import PamController.PamSettings; -import PamController.settings.output.xml.PamguardXMLWriter; import PamUtils.PamCalendar; import PamguardMVC.PamDataBlock; -import PamguardMVC.PamDataUnit; -import PamguardMVC.dataSelector.DataSelector; -import dbxml.uploader.Importer; import metadata.MetaDataContol; import metadata.deployment.DeploymentData; import nilus.Deployment; @@ -35,15 +19,13 @@ import tethys.dbxml.DBXMLConnect; import tethys.dbxml.TethysException; import tethys.deployment.DeploymentHandler; import tethys.deployment.DeploymentOverview; -import tethys.deployment.DeploymentRecoveryPair; import tethys.deployment.RecordingPeriod; -import tethys.detection.DetectionGranularity; -import tethys.detection.DetectionGranularity.GRANULARITY; import tethys.detection.DetectionsHandler; -import tethys.pamdata.TethysDataProvider; -import tethys.pamdata.TethysSchema; /** + * No longer used. This was purely a test class used for making the first couple + * of test connections and exports to Tethys. Can probably delete. + * * Class sitting at the centre of all operations. It will talk to PAMGuard * objects to get schemas and data and talk to the database connection to move * data out (and possibly in). Eventually, a lot of the functionality in here @@ -55,6 +37,7 @@ import tethys.pamdata.TethysSchema; * @author dg50 * */ +@Deprecated public class TethysExporter { private TethysControl tethysControl; @@ -62,7 +45,7 @@ public class TethysExporter { private DBXMLConnect dbxmlConnect; - public TethysExporter(TethysControl tethysControl, TethysExportParams tethysExportParams) { + private TethysExporter(TethysControl tethysControl, TethysExportParams tethysExportParams) { this.tethysControl = tethysControl; this.tethysExportParams = tethysExportParams; dbxmlConnect = new DBXMLConnect(tethysControl); @@ -85,7 +68,7 @@ public class TethysExporter { */ // return false; // } - + /** * Doug populate instrument fields - may need to add a few things. Marie to * define what we mean by instrument. Instrument names probably need to be added @@ -206,7 +189,7 @@ public class TethysExporter { DetectionsHandler detectionsHandler = new DetectionsHandler(tethysControl); ArrayList allDataBlocks = PamController.getInstance().getDataBlocks(); // /** -// * Outer loop is through deployemnt documents. Will then export detections within each +// * Outer loop is through deployemnt documents. Will then export detections within each // * deployment detector by detector // */ // for (Deployment aDeployment : deploymentDocs) { @@ -215,7 +198,7 @@ public class TethysExporter { // if (streamExportParams == null || !streamExportParams.selected) { // continue; // not interested in this one. // } -// detectionsHandler.exportDetections(aDataBlock, aDeployment, +// detectionsHandler.exportDetections(aDataBlock, aDeployment, // new DetectionGranularity(GRANULARITY.TIME, 3600*12), tethysExportParams, streamExportParams); // } // } diff --git a/src/tethys/output/swing/TethysExportDialog.java b/src/tethys/output/swing/TethysExportDialog.java index ae86b315..46a6c690 100644 --- a/src/tethys/output/swing/TethysExportDialog.java +++ b/src/tethys/output/swing/TethysExportDialog.java @@ -112,7 +112,7 @@ public class TethysExportDialog extends PamDialog { ArrayList sets = new ArrayList<>(); ArrayList allDataBlocks = PamController.getInstance().getDataBlocks(); for (PamDataBlock aDataBlock : allDataBlocks) { - if (aDataBlock.getTethysDataProvider() != null) { + if (aDataBlock.getTethysDataProvider(tethysControl) != null) { sets.add(new DataStreamSet(aDataBlock)); } } diff --git a/src/tethys/pamdata/AutoTethysProvider.java b/src/tethys/pamdata/AutoTethysProvider.java index 57b254b0..23a465a0 100644 --- a/src/tethys/pamdata/AutoTethysProvider.java +++ b/src/tethys/pamdata/AutoTethysProvider.java @@ -31,6 +31,7 @@ import nilus.DetectionEffortKind; import nilus.SpeciesIDType; import tethys.TethysControl; import tethys.TethysTimeFuncs; +import tethys.detection.DetectionsHandler; import tethys.niluswraps.PDeployment; import tethys.output.StreamExportParams; import tethys.output.TethysExportParams; @@ -62,8 +63,10 @@ public class AutoTethysProvider implements TethysDataProvider { private PamDataBlock pamDataBlock; private PamProcess pamProcess; private PamControlledUnit pamControlledUnit; + private TethysControl tethysControl; - public AutoTethysProvider(PamDataBlock pamDataBlock) { + public AutoTethysProvider(TethysControl tethysControl, PamDataBlock pamDataBlock) { + this.tethysControl = tethysControl; this.pamDataBlock = pamDataBlock; pamProcess = pamDataBlock.getParentProcess(); pamControlledUnit = pamProcess.getPamControlledUnit(); @@ -135,7 +138,7 @@ public class AutoTethysProvider implements TethysDataProvider { Object settings = pamSettings.getSettingsReference(); TethysParameterPacker paramPacker = null; try { - paramPacker = new TethysParameterPacker(); + paramPacker = new TethysParameterPacker(tethysControl); } catch (JAXBException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -425,6 +428,22 @@ public class AutoTethysProvider implements TethysDataProvider { kind.getSpeciesId().setValue(BigInteger.valueOf(mapItem.getItisCode())); kind.getGranularity().setValue(exportParams.granularity); +// nilus.DetectionEffortKind.Parameters granularityParams = kind.getParameters(); + switch (exportParams.granularity) { + case BINNED: + kind.getGranularity().setBinSizeM(exportParams.binDurationS/60.); + long firstBin = DetectionsHandler.roundDownBinStart(pDeployment.getAudioStart(), (long) (exportParams.binDurationS*1000)); + kind.getGranularity().setFirstBinStart(TethysTimeFuncs.xmlGregCalFromMillis(firstBin)); + break; + case CALL: + break; + case ENCOUNTER: + kind.getGranularity().setEncounterGapM(exportParams.encounterGapS/60.); + break; + case GROUPED: + break; + + } kind.setCall(mapItem.getCallType()); diff --git a/src/tethys/pamdata/TethysParameterPacker.java b/src/tethys/pamdata/TethysParameterPacker.java index 55535594..b663db20 100644 --- a/src/tethys/pamdata/TethysParameterPacker.java +++ b/src/tethys/pamdata/TethysParameterPacker.java @@ -14,6 +14,7 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.dom.DOMResult; +import org.docx4j.model.listnumbering.NumberFormatLowerLetter; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -25,7 +26,10 @@ import PamModel.parametermanager.PamParameterData; import PamModel.parametermanager.PamParameterSet; import PamguardMVC.PamDataBlock; import PamguardMVC.PamProcess; +import PamguardMVC.dataSelector.DataSelectParams; +import PamguardMVC.dataSelector.DataSelector; import nilus.MarshalXML; +import tethys.TethysControl; /** * Functions to pack up a PAMGuard parameters object into the correct format @@ -74,12 +78,15 @@ public class TethysParameterPacker { private PamguardXMLWriter xmlWriter; + private TethysControl tethysControl; + /** * @throws JAXBException * */ - public TethysParameterPacker() throws JAXBException { + public TethysParameterPacker(TethysControl tethysControl) throws JAXBException { super(); + this.tethysControl = tethysControl; try { marshaller = new MarshalXML(); } catch (JAXBException e) { @@ -117,12 +124,43 @@ public class TethysParameterPacker { if (parameterSet == null) { return null; } -// Document document = null; -// try { -// document = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument(); -// } catch (ParserConfigurationException e1) { -// e1.printStackTrace(); -// } + // get the XML writer ready for a new export ... + xmlWriter.setExcludeDisplaySettings(true); + xmlWriter.makeSettingsList(); + + /** + * first do the data filter. I can't see any way of doing this + * without creating a doc as was in the helper example. + */ + QName qnamef = new QName(MarshalXML.schema, "datafilter", "ty"); + JAXBElement jaxelf = new JAXBElement( + qnamef, String.class, parameterSet.getParentObject().getClass().getCanonicalName()); + Document docf = null; + try { + docf = marshaller.marshalToDOM(jaxelf); + } catch (JAXBException | ParserConfigurationException e1) { + e1.printStackTrace(); + } + Element elf = docf.getDocumentElement(); + elList.add(elf);/** + * Is there a data filter ? If so, write it's + * XML parameters out here. + */ + DataSelector dataSelector = pamDataBlock.getDataSelector(tethysControl.getDataSelectName(), false); + if (dataSelector != null) { + DataSelectParams filterParams = dataSelector.getParams(); + if (filterParams != null) { + Element pEl = xmlWriter.writeObjectData(docf, elf, filterParams, null); +// if (pEl != null) { +//// filterEl.appendChild(pEl); +// elf.appendChild(filterEl); +// } + } + } + + + + QName qname = new QName(MarshalXML.schema, "parameters", "ty"); JAXBElement jaxel = new JAXBElement( qname, String.class, parameterSet.getParentObject().getClass().getCanonicalName()); @@ -133,18 +171,14 @@ public class TethysParameterPacker { e1.printStackTrace(); } Element el = doc.getDocumentElement(); - - for (PamParameterData pamParam : parameterSet.getParameterCollection()) { - try { - Object paramData = pamParam.getData(); - boolean ok = createElement(doc, el, paramData, pamParam, objectHierarchy); - } catch (IllegalArgumentException | IllegalAccessException e) { - e.printStackTrace(); - } - } elList.add(el); - xmlWriter.setExcludeDisplaySettings(true); - xmlWriter.makeSettingsList(); + + + + /** + * Now get the chain of PAMGuard modules for the current detector and for + * all upstream modules. + */ ArrayList moduleChain = getParentChain(pamDataBlock); for (PamControlledUnit pcu : moduleChain) { if (pcu instanceof PamSettings == false) { diff --git a/src/tethys/swing/DatablockDetectionsPanel.java b/src/tethys/swing/DatablockDetectionsPanel.java index be174a5e..2e2ebb40 100644 --- a/src/tethys/swing/DatablockDetectionsPanel.java +++ b/src/tethys/swing/DatablockDetectionsPanel.java @@ -20,6 +20,8 @@ import PamView.tables.SwingTableColumnWidths; import PamguardMVC.PamDataBlock; import nilus.Detections; import tethys.TethysControl; +import tethys.TethysState; +import tethys.TethysState.StateType; import tethys.dbxml.TethysException; import tethys.detection.StreamDetectionsSummary; import tethys.niluswraps.PDetections; @@ -75,6 +77,13 @@ public class DatablockDetectionsPanel extends TethysGUIPanel implements StreamTa tableModel.fireTableDataChanged(); } + @Override + public void updateState(TethysState tethysState) { + if (dataBlock != null) { + selectDataBlock(dataBlock); + } + } + private class MouseActions extends MouseAdapter { @Override @@ -137,6 +146,7 @@ public class DatablockDetectionsPanel extends TethysGUIPanel implements StreamTa } catch (TethysException e) { getTethysControl().showException(e); } + getTethysControl().exportedDetections(dataBlock); selectDataBlock(dataBlock); // force table update. } diff --git a/src/tethys/swing/DatablockSynchPanel.java b/src/tethys/swing/DatablockSynchPanel.java index 212eadf2..1c1131b5 100644 --- a/src/tethys/swing/DatablockSynchPanel.java +++ b/src/tethys/swing/DatablockSynchPanel.java @@ -28,10 +28,11 @@ import PamguardMVC.PamDataBlock; import dataMap.OfflineDataMap; import tethys.TethysControl; import tethys.TethysState; +import tethys.TethysStateObserver; import tethys.output.DatablockSynchInfo; import tethys.species.DataBlockSpeciesManager; -public class DatablockSynchPanel extends TethysGUIPanel { +public class DatablockSynchPanel extends TethysGUIPanel { public JPanel mainPanel; @@ -133,6 +134,7 @@ public class DatablockSynchPanel extends TethysGUIPanel { @Override public void updateState(TethysState tethysState) { synchTableModel.fireTableDataChanged(); + selectRow(); } public void addTableObserver(StreamTableObserver observer) { diff --git a/src/tethys/swing/export/AlgorithmCard.java b/src/tethys/swing/export/AlgorithmCard.java index fd82e483..ff949d2b 100644 --- a/src/tethys/swing/export/AlgorithmCard.java +++ b/src/tethys/swing/export/AlgorithmCard.java @@ -20,7 +20,7 @@ public class AlgorithmCard extends ExportWizardCard { private JTextField method, software, version, supportSoftware; - public AlgorithmCard(TethysControl tethysControl, PamDataBlock dataBlock) { + public AlgorithmCard(DetectionsExportWizard detectionsExportWizard, TethysControl tethysControl, PamDataBlock dataBlock) { super(tethysControl, "Algorithm", dataBlock); setBorder(new TitledBorder("Algorithm details")); method = new JTextField(40); diff --git a/src/tethys/swing/export/DescriptionCard.java b/src/tethys/swing/export/DescriptionCard.java index 13b13ee3..c2bacaef 100644 --- a/src/tethys/swing/export/DescriptionCard.java +++ b/src/tethys/swing/export/DescriptionCard.java @@ -10,7 +10,7 @@ public class DescriptionCard extends ExportWizardCard { private DescriptionTypePanel descriptionPanel; - public DescriptionCard(TethysControl tethysControl, PamDataBlock dataBlock) { + public DescriptionCard(DetectionsExportWizard detectionsExportWizard, TethysControl tethysControl, PamDataBlock dataBlock) { super(tethysControl, "Description", dataBlock); this.setLayout(new BorderLayout()); descriptionPanel = new DescriptionTypePanel("Description data", true, true, true); diff --git a/src/tethys/swing/export/DetectionsExportWizard.java b/src/tethys/swing/export/DetectionsExportWizard.java index d04a8550..94ece212 100644 --- a/src/tethys/swing/export/DetectionsExportWizard.java +++ b/src/tethys/swing/export/DetectionsExportWizard.java @@ -48,10 +48,10 @@ public class DetectionsExportWizard extends PamDialog { cardPanel = new JPanel(cardLayout); mainPanel.add(BorderLayout.CENTER, cardPanel); - addCard(algorithmCard = new AlgorithmCard(tethysControl, dataBlock)); - addCard(granularityCard = new GranularityCard(tethysControl, dataBlock)); - addCard(descriptionCard = new DescriptionCard(tethysControl, dataBlock)); - addCard(exportWorkerCard = new ExportWorkerCard(tethysControl, dataBlock)); + addCard(algorithmCard = new AlgorithmCard(this, tethysControl, dataBlock)); + addCard(granularityCard = new GranularityCard(this, tethysControl, dataBlock)); + addCard(descriptionCard = new DescriptionCard(this, tethysControl, dataBlock)); + addCard(exportWorkerCard = new ExportWorkerCard(this, tethysControl, dataBlock)); cardLayout.first(cardPanel); @@ -98,6 +98,10 @@ public class DetectionsExportWizard extends PamDialog { cardLayout.previous(cardPanel); enableControls(); } + + public JButton getPreviousButton() { + return prevButton; + } @Override public boolean getParams() { diff --git a/src/tethys/swing/export/ExportWorkerCard.java b/src/tethys/swing/export/ExportWorkerCard.java index 3b4f73ee..bfe0c983 100644 --- a/src/tethys/swing/export/ExportWorkerCard.java +++ b/src/tethys/swing/export/ExportWorkerCard.java @@ -36,8 +36,11 @@ public class ExportWorkerCard extends ExportWizardCard implements DetectionExpor private StreamExportParams streamExportParams; - public ExportWorkerCard(TethysControl tethysControl, PamDataBlock dataBlock) { + private DetectionsExportWizard detectionsExportWizard; + + public ExportWorkerCard(DetectionsExportWizard detectionsExportWizard, TethysControl tethysControl, PamDataBlock dataBlock) { super(tethysControl, "Export", dataBlock); + this.detectionsExportWizard = detectionsExportWizard; setLayout(new BorderLayout()); setBorder(new TitledBorder("Export data")); JPanel exPanel = new PamNorthPanel(new GridBagLayout()); @@ -146,6 +149,8 @@ public class ExportWorkerCard extends ExportWizardCard implements DetectionExpor break; case DetectionExportProgress.STATE_COMPLETE: progressText.setText("Export complete"); + detectionsExportWizard.getCancelButton().setText("Close"); + detectionsExportWizard.getPreviousButton().setEnabled(false); break; case DetectionExportProgress.STATE_WRITING: progressText.setText("Writing to Tethys: " + progress.currentDetections.getId()); diff --git a/src/tethys/swing/export/GranularityCard.java b/src/tethys/swing/export/GranularityCard.java index c7f3c4b8..a8fa539e 100644 --- a/src/tethys/swing/export/GranularityCard.java +++ b/src/tethys/swing/export/GranularityCard.java @@ -34,17 +34,22 @@ import tethys.output.StreamExportParams; public class GranularityCard extends ExportWizardCard { private JRadioButton[] granularities; - + private JTextArea dataSelectionText; - - private JTextField binLength, encounterGap; + + private JTextField binLength, minCalls, encounterGap; private DataSelector dataSelector; - - public GranularityCard(TethysControl tethysControl, PamDataBlock dataBlock) { + + private DetectionsExportWizard detectionsExportWizard; + + private int encounterIndex, binnedIndex; + + public GranularityCard(DetectionsExportWizard detectionsExportWizard, TethysControl tethysControl, PamDataBlock dataBlock) { super(tethysControl, "Granularity", dataBlock); + this.detectionsExportWizard = detectionsExportWizard; setLayout(new BoxLayout(this, BoxLayout.Y_AXIS)); - + // granularity GranularityEnumType[] grans = GranularityEnumType.values(); granularities = new JRadioButton[grans.length]; @@ -52,34 +57,41 @@ public class GranularityCard extends ExportWizardCard { GridBagConstraints c = new PamGridBagContraints(); granPanel.setBorder(new TitledBorder("Granularity")); ButtonGroup granGroup = new ButtonGroup(); + GranularityChange gc = new GranularityChange(); for (int i = 0; i < grans.length; i++) { c.gridx = 0; granularities[i] = new JRadioButton(PGranularityType.prettyString(grans[i])); granularities[i].setToolTipText(PGranularityType.toolTip(grans[i])); + granularities[i].addActionListener(gc); granPanel.add(granularities[i], c); granGroup.add(granularities[i]); if (grans[i] == GranularityEnumType.BINNED) { + binnedIndex = i; c.gridx++; granPanel.add(new JLabel(" bin duration ", JLabel.RIGHT), c); c.gridx++; granPanel.add(binLength = new JTextField(5), c); c.gridx++; - granPanel.add(new JLabel(" (s) ", JLabel.LEFT), c); - + granPanel.add(new JLabel("(s), min Calls", JLabel.LEFT), c); + c.gridx++; + granPanel.add(minCalls = new JTextField(5), c); + binLength.setToolTipText("Time bin duration in seconds"); + minCalls.setToolTipText("Minimum number of calls for a bin to be output"); } if (grans[i] == GranularityEnumType.ENCOUNTER) { + encounterIndex = i; c.gridx++; granPanel.add(new JLabel(" min gap ", JLabel.RIGHT), c); c.gridx++; granPanel.add(encounterGap = new JTextField(5), c); c.gridx++; - granPanel.add(new JLabel(" (s) ", JLabel.LEFT), c); - + granPanel.add(new JLabel("(s) ", JLabel.LEFT), c); + encounterGap.setToolTipText("Minimum gap between separate encounters"); } c.gridy++; } this.add(granPanel); - + // data selection dataSelector = dataBlock.getDataSelector(tethysControl.getDataSelectName(), false); if (dataSelector != null) { @@ -101,7 +113,22 @@ public class GranularityCard extends ExportWizardCard { dataPanel.add(BorderLayout.CENTER, sp); this.add(dataPanel); } - + + } + + private class GranularityChange implements ActionListener { + + @Override + public void actionPerformed(ActionEvent e) { + enableControls(); + } + + } + + private void enableControls() { + binLength.setEnabled(granularities[binnedIndex].isSelected()); + minCalls.setEnabled(granularities[binnedIndex].isSelected()); + encounterGap.setEnabled(granularities[encounterIndex].isSelected()); } protected void newDataSelection() { @@ -129,7 +156,29 @@ public class GranularityCard extends ExportWizardCard { break; } } - + if (streamExportParams.granularity == GranularityEnumType.BINNED) { + try { + streamExportParams.binDurationS = Double.valueOf(binLength.getText()); + } + catch (NumberFormatException e) { + return detectionsExportWizard.showWarning("Invalid bin duration parameter"); + } + try { + streamExportParams.minBinCount = Integer.valueOf(minCalls.getText()); + } + catch (NumberFormatException e) { + return detectionsExportWizard.showWarning("Invalid minimum call count"); + } + } + if (streamExportParams.granularity == GranularityEnumType.ENCOUNTER) { + try { + streamExportParams.encounterGapS = Double.valueOf(encounterGap.getText()); + } + catch (NumberFormatException e) { + return detectionsExportWizard.showWarning("Invalid encounter gap parameter"); + } + } + return streamExportParams.granularity != null; } @@ -139,7 +188,11 @@ public class GranularityCard extends ExportWizardCard { for (int i = 0; i < grans.length; i++) { granularities[i].setSelected(streamExportParams.granularity == grans[i]); } + binLength.setText(String.format("%3.1f", streamExportParams.binDurationS)); + minCalls.setText(String.format("%d", streamExportParams.minBinCount)); + encounterGap.setText(String.format("%3.1f", streamExportParams.encounterGapS)); newDataSelection(); + enableControls(); } }