nordic-dev.net/PAMGuard
nordic-dev.net/PAMGuard
2
0
Fork 0
mirror of https://github.com/PAMGuard/PAMGuard.git synced 2024-11-21 22:52:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Trying to Merge with KB Changes

Browse Source
This commit is contained in:
Douglas Gillespie 2023-03-14 17:28:13 +00:00
parent 2dd870956a
commit 103c050d0c
7 changed files with 680 additions and 306 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Array/Hydrophone.java
View File

@ -333,7 +333,7 @@ public class Hydrophone implements Serializable, Cloneable, ManagedParameters {
/**
* @return Returns the coordinate.
*/
protected double[] getCoordinates() {
public double[] getCoordinates() {
return Arrays.copyOf(coordinate,3);
}

429
src/tethys/deployment/DeploymentHandler.java Normal file
View File

@ -0,0 +1,429 @@
package tethys.deployment;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
import javax.xml.bind.JAXBException;
import org.apache.commons.beanutils.converters.BigIntegerConverter;
import Acquisition.AcquisitionControl;
import Acquisition.AcquisitionParameters;
import Acquisition.DaqStatusDataUnit;
import Acquisition.DaqSystem;
import Array.ArrayManager;
import Array.Hydrophone;
import Array.PamArray;
import PamController.PamControlledUnit;
import PamController.PamController;
import PamUtils.PamCalendar;
import PamUtils.PamUtils;
import PamguardMVC.PamDataBlock;
import nilus.Audio;
import nilus.ChannelInfo;
import nilus.ChannelInfo.Sampling;
import nilus.ChannelInfo.Sampling.Regimen;
import nilus.Deployment;
import nilus.Deployment.SamplingDetails;
import nilus.Deployment.Sensors;
import nilus.DeploymentRecoveryDetails;
import nilus.GeometryTypeM;
import nilus.Helper;
import nilus.MarshalXML;
import pamMaths.PamVector;
import tethys.TethysLocationFuncs;
import tethys.TethysTimeFuncs;
/**
* Functions to gather data for the deployment document from all around PAMGuard.
* @author dg50
*
*/
public class DeploymentHandler {
/**
* Get an overview of all the deployments.
* @return
*/
public DeploymentOverview createOverview() {
// first find an acquisition module.
PamControlledUnit aModule = PamController.getInstance().findControlledUnit(AcquisitionControl.class, null);
if (aModule instanceof AcquisitionControl == false) {
// will return if it's null. Impossible for it to be the wrong type.
// but it's good practice to check anyway before casting.
return null;
}
// cast it to the right type.
AcquisitionControl daqControl = (AcquisitionControl) aModule;
AcquisitionParameters daqParams = daqControl.getAcquisitionParameters();
/**
* The daqParams class has most of what we need about the set up in terms of sample rate,
* number of channels, instrument type, ADC input range (part of calibration), etc.
* It also has a hydrophone list, which maps the input channel numbers to the hydrophon numbers.
* Realistically, this list is always 0,1,2,etc or it goes horribly wrong !
*/
// so write functions here to get information from the daqParams.
System.out.printf("Sample regime: %s input with rate %3.1fHz, %d channels, gain %3.1fdB, ADCp-p %3.1fV\n", daqParams.getDaqSystemType(),
daqParams.getSampleRate(), daqParams.getNChannels(), daqParams.preamplifier.getGain(), daqParams.voltsPeak2Peak);
/**
* then there is the actual sampling. This is a bit harder to find. I thought it would be in the data map
* but the datamap is a simple count of what's in the databasase which is not quite what we want.
* we're going to have to query the database to get more detailed informatoin I think.
* I'll do that here for now, but we may want to move this when we better organise the code.
* It also seems that there are 'bad' dates in the database when it starts new files, which are the date
* data were analysed at. So we really need to check the start and stop records only.
*/
PamDataBlock<DaqStatusDataUnit> daqInfoDataBlock = daqControl.getAcquisitionProcess().getDaqStatusDataBlock();
// just load everything. Probably OK for the acqusition, but will bring down
daqInfoDataBlock.loadViewerData(0, Long.MAX_VALUE, null);
ArrayList<DaqStatusDataUnit> allStatusData = daqInfoDataBlock.getDataCopy();
ArrayList<RecordingPeriod> tempPeriods = null;
if (allStatusData == null || allStatusData.size() == 0) {
System.out.println("Data appear to have no logged recording periods. Try to extract from raw audio ...");
tempPeriods = extractTimesFromFiles(daqControl);
}
else {
tempPeriods = extractTimesFromStatus(allStatusData);
}
if (tempPeriods == null || tempPeriods.size() == 0) {
System.out.println("Data appear to have no logged recording periods available either from the database or the raw recordings.");
}
int nPeriods = tempPeriods.size();
// now go through those and merge into longer periods where there is no gap between files.
ListIterator<RecordingPeriod> iterator = tempPeriods.listIterator();
RecordingPeriod prevPeriod = null;
while (iterator.hasNext()) {
RecordingPeriod nextPeriod = iterator.next();
long nextDur = nextPeriod.getRecordStop()-nextPeriod.getRecordStart();
if (nextDur == 0) {
continue;
}
if (prevPeriod != null) {
long gap = nextPeriod.getRecordStart() - prevPeriod.getRecordStop();
long prevDur = prevPeriod.getRecordStop()-prevPeriod.getRecordStart();
if (gap < 3 || gap < prevDur/50) {
// ignoring up to 3s gap or a sample error < 2%.Dunno if this is sensible or not.
prevPeriod.setRecordStop(nextPeriod.getRecordStop());
iterator.remove();
nextPeriod = prevPeriod;
}
}
prevPeriod = nextPeriod;
}
System.out.printf("Data have %d distinct files, but only %d distinct recording periods\n", nPeriods, tempPeriods.size());
DutyCycleInfo dutyCycleinfo = assessDutyCycle(tempPeriods);
DeploymentOverview deploymentOverview = new DeploymentOverview(false, tempPeriods);
return deploymentOverview;
// find the number of times it started and stopped ....
// System.out.printf("Input map of sound data indicates data from %s to %s with %d starts and %d stops over %d files\n",
// PamCalendar.formatDateTime(dataStart), PamCalendar.formatDateTime(dataEnd), nStart, nStop, nFile+1);
// now work out where there are genuine gaps and make up a revised list of recording periods.
}
/**
* Work out whether or not the data are evenly duty cycled by testing the
* distributions of on and off times.
* @param tempPeriods
* @return
*/
private DutyCycleInfo assessDutyCycle(ArrayList<RecordingPeriod> tempPeriods) {
int n = tempPeriods.size();
if (n < 2) {
return null;
}
double[] ons = new double[n-1]; // ignore the last one since it may be artificially shortened which is OK
double[] gaps = new double[n-1];
for (int i = 0; i < n; i++) {
ons[i] = tempPeriods.get(i).getDuration();
}
return null;
}
private ArrayList<RecordingPeriod> extractTimesFromStatus(ArrayList<DaqStatusDataUnit> allStatusData) {
ArrayList<RecordingPeriod> tempPeriods = new ArrayList<>();
long dataStart = Long.MAX_VALUE;
long dataEnd = Long.MIN_VALUE;
Long lastStart = null;
int nStart = 0;
int nStop = 0;
int nFile = 0;
for (DaqStatusDataUnit daqStatus : allStatusData) {
switch (daqStatus.getStatus()) {
case "Start":
nStart++;
dataStart = Math.min(dataStart, daqStatus.getTimeMilliseconds());
lastStart = daqStatus.getTimeMilliseconds();
break;
case "Stop":
nStop++;
dataEnd = Math.max(dataEnd, daqStatus.getEndTimeInMilliseconds());
long lastEnd = daqStatus.getEndTimeInMilliseconds();
if (lastStart != null) {
tempPeriods.add(new RecordingPeriod(lastStart, lastEnd));
}
lastStart = null;
break;
case "NextFile":
nFile++;
break;
}
}
return tempPeriods;
}
private ArrayList<RecordingPeriod> extractTimesFromFiles(AcquisitionControl daqControl) {
// TODO Auto-generated method stub
return null;
}
//in each channel
public ArrayList<DeploymentRecoveryPair> getDeployments() {
DeploymentOverview recordingOverview = createOverview();
// first find an acquisition module.
PamControlledUnit aModule = PamController.getInstance().findControlledUnit(AcquisitionControl.class, null);
if (aModule instanceof AcquisitionControl == false) {
// will return if it's null. Impossible for it to be the wrong type.
// but it's good practice to check anyway before casting.
return null;
}
// cast it to the right type.
AcquisitionControl daqControl = (AcquisitionControl) aModule;
AcquisitionParameters daqParams = daqControl.getAcquisitionParameters();
/**
* The daqParams class has most of what we need about the set up in terms of sample rate,
* number of channels, instrument type, ADC input range (part of calibration), etc.
* It also has a hydrophone list, which maps the input channel numbers to the hydrophon numbers.
* Realistically, this list is always 0,1,2,etc or it goes horribly wrong !
*/
// so write functions here to get information from the daqParams.
System.out.printf("Sample regime: %s input with rate %3.1fHz, %d channels, gain %3.1fdB, ADCp-p %3.1fV\n", daqParams.getDaqSystemType(),
daqParams.getSampleRate(), daqParams.getNChannels(), daqParams.preamplifier.getGain(), daqParams.voltsPeak2Peak);
/**
* then there is the actual sampling. This is a bit harder to find. I thought it would be in the data map
* but the datamap is a simple count of what's in the databasase which is not quite what we want.
* we're going to have to query the database to get more detailed informatoin I think.
* I'll do that here for now, but we may want to move this when we better organise the code.
* It also seems that there are 'bad' dates in the database when it starts new files, which are the date
* data were analysed at. So we really need to check the start and stop records only.
*/
PamDataBlock<DaqStatusDataUnit> daqInfoDataBlock = daqControl.getAcquisitionProcess().getDaqStatusDataBlock();
// just load everything. Probably OK for the acqusition, but will bring down
daqInfoDataBlock.loadViewerData(0, Long.MAX_VALUE, null);
ArrayList<DaqStatusDataUnit> allStatusData = daqInfoDataBlock.getDataCopy();
long dataStart = Long.MAX_VALUE;
long dataEnd = Long.MIN_VALUE;
if (allStatusData != null && allStatusData.size() > 0) {
// find the number of times it started and stopped ....
int nStart = 0, nStop = 0, nFile=0;
for (DaqStatusDataUnit daqStatus : allStatusData) {
switch (daqStatus.getStatus()) {
case "Start":
nStart++;
dataStart = Math.min(dataStart, daqStatus.getTimeMilliseconds());
break;
case "Stop":
nStop++;
dataEnd = Math.max(dataEnd, daqStatus.getEndTimeInMilliseconds());
break;
case "NextFile":
nFile++;
break;
}
}
System.out.printf("Input map of sound data indicates data from %s to %s with %d starts and %d stops over %d files\n",
PamCalendar.formatDateTime(dataStart), PamCalendar.formatDateTime(dataEnd), nStart, nStop, nFile+1);
}
// // and we find the datamap within that ...
// OfflineDataMap daqMap = daqInfoDataBlock.getOfflineDataMap(DBControlUnit.findDatabaseControl());
// if (daqMap != null) {
// // iterate through it.
// long dataStart = daqMap.getFirstDataTime();
// long dataEnd = daqMap.getLastDataTime();
// List<OfflineDataMapPoint> mapPoints = daqMap.getMapPoints();
// System.out.printf("Input map of sound data indicates data from %s to %s with %d individual files\n",
// PamCalendar.formatDateTime(dataStart), PamCalendar.formatDateTime(dataEnd), mapPoints.size());
// /*
// * clearly in the first database I've been looking at of Tinas data, this is NOT getting sensible start and
// * end times. Print them out to see what's going on.
// */
//// for ()
// }
DeploymentRecoveryPair pair = new DeploymentRecoveryPair();
DeploymentRecoveryDetails deployment = new DeploymentRecoveryDetails();
DeploymentRecoveryDetails recovery = new DeploymentRecoveryDetails();
pair.deploymentDetails = deployment;
pair.recoveryDetails = recovery;
deployment.setTimeStamp(TethysTimeFuncs.xmlGregCalFromMillis(dataStart));
deployment.setAudioTimeStamp(TethysTimeFuncs.xmlGregCalFromMillis(dataStart));
recovery.setTimeStamp(TethysTimeFuncs.xmlGregCalFromMillis(dataEnd));
recovery.setAudioTimeStamp(TethysTimeFuncs.xmlGregCalFromMillis(dataEnd));
ArrayList<DeploymentRecoveryPair> drPairs = new ArrayList<>();
drPairs.add(pair);
return drPairs;
}
public Deployment createDeploymentDocument(int i, DeploymentRecoveryPair drd) {
Deployment deployment = new Deployment();
String id = String.format("%d", i);
deployment.setId(id);
deployment.setDeploymentId(i);
deployment.setDeploymentDetails(drd.deploymentDetails);
deployment.setRecoveryDetails(drd.recoveryDetails);
TethysLocationFuncs.getTrackAndPositionData(deployment);
getSamplingDetails(deployment);
getSensorDetails(deployment);
/**
* Stuff that may need to be put into the UI:
* Audio: can easily get current loc of raw and binary data, but may need to override these. I think
* this may be for the export UI ?
* Tracks: trackline information. General problem in PAMGUard.
*/
return deployment;
}
private boolean getSensorDetails(Deployment deployment) {
PamArray array = ArrayManager.getArrayManager().getCurrentArray();
Sensors sensors = new Sensors();
List<Audio> audioList = sensors.getAudio();
ArrayList<Hydrophone> phones = array.getHydrophoneArray();
int iPhone = 0;
long timeMillis = TethysTimeFuncs.millisFromGregorianXML(deployment.getDeploymentDetails().getAudioTimeStamp());
Helper nilusHelper = null;
try {
nilusHelper = new Helper();
} catch (JAXBException e) {
e.printStackTrace();
}
for (Hydrophone aPhone : phones) {
PamVector hydLocs = array.getAbsHydrophoneVector(iPhone, timeMillis);
Audio audio = new Audio();
audio.setNumber(BigInteger.valueOf(iPhone));
audio.setSensorId(String.format("Hydrophone %d", iPhone)); // shold replace with serial number if it exists.
GeometryTypeM geom = new GeometryTypeM();
geom.setXM(hydLocs.getCoordinate(0));
geom.setYM(hydLocs.getCoordinate(1));
geom.setZM(hydLocs.getCoordinate(2));
// Geometry geom = new Geometry();
// audio.setGeometry(geom);
//// nilusHelper.
// List<Serializable> geomCont = geom.getContent();
// for (int iCoord = 0; iCoord < 3; iCoord++) {
// geom.getContent().add(Double.valueOf(hydLocs.getCoordinate(iCoord)));
// }
// try {
// MarshalXML mXML = new MarshalXML();
// mXML.marshal(geom);
// } catch (JAXBException e) {
// // TODO Auto-generated catch block
// e.printStackTrace();
// }
/**
* Need to be able to add the values from hydLocs to the geometry object, but can't.
*/
audioList.add(audio);
iPhone++;
}
try {
MarshalXML mXML = new MarshalXML();
mXML.marshal(sensors);
} catch (JAXBException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
deployment.setSensors(sensors);
return true;
}
/**
* Fill in the sampling details in a Deployment document.
* @param deployment
*/
private boolean getSamplingDetails(Deployment deployment) {
SamplingDetails samplingDetails = new SamplingDetails();
// this is basically going to be a list of almost identical channel information
// currently just for the first acquisition. May extend to more.
AcquisitionControl daq = (AcquisitionControl) PamController.getInstance().findControlledUnit(AcquisitionControl.class, null);
if (daq == null) {
return false;
}
DaqSystem system = daq.findDaqSystem(null);
AcquisitionParameters daqParams = daq.acquisitionParameters;
int nChan = daqParams.nChannels;
float fs = daqParams.sampleRate;
int[] hydroMap = daqParams.getHydrophoneList();
int[] inputMap = daqParams.getHardwareChannelList();
double vp2p = daqParams.getVoltsPeak2Peak();
List<ChannelInfo> channelInfos = samplingDetails.getChannel();
for (int i = 0; i < nChan; i++) {
ChannelInfo channelInfo = new ChannelInfo();
channelInfo.setStart(deployment.getDeploymentDetails().getAudioTimeStamp());
channelInfo.setEnd(deployment.getRecoveryDetails().getAudioTimeStamp());
BigIntegerConverter biCon = new BigIntegerConverter();
BigInteger chanNum = BigInteger.valueOf((long) i);
channelInfo.setChannelNumber(chanNum);
if (hydroMap != null) {
channelInfo.setSensorNumber(hydroMap[i]);
}
else {
channelInfo.setSensorNumber(i);
}
/*
* Gain - may have to cycle through and see if this ever changes (or
* if was recorded that it changed which may not be the same!)
*/
ChannelInfo.Gain gain = new ChannelInfo.Gain();
List<nilus.ChannelInfo.Gain.Regimen> gainList = gain.getRegimen();
nilus.ChannelInfo.Gain.Regimen aGain = new nilus.ChannelInfo.Gain.Regimen();
aGain.setGainDB(daqParams.getPreamplifier().getGain());
channelInfo.setGain(gain);
Sampling sampling = new Sampling();
List<Regimen> regimens = sampling.getRegimen();
Sampling.Regimen regimen = new Sampling.Regimen();
regimen.setTimeStamp(deployment.getDeploymentDetails().getAudioTimeStamp());
regimen.setSampleRateKHz(fs/1000.);
if (system != null) {
regimen.setSampleBits(system.getSampleBits());
}
regimens.add(regimen);
channelInfo.setSampling(sampling);
channelInfos.add(channelInfo);
/**
* Need something about duty cycling. this is probably something that will have to be added
* earlier to a wrapper around the Deployment class.
*/
}
deployment.setSamplingDetails(samplingDetails);
return true;
}
}

50
src/tethys/deployment/DeploymentOverview.java Normal file
View File

@ -0,0 +1,50 @@
package tethys.deployment;
import java.util.ArrayList;
/**
* Class to give a general overview of all the effort in PAMGuard which will form the
* basis for Deployment Documents.
* This will tell us if data were on a regular duty cycle or more adhoc and also provide
* a list of all start and end times for these periods.
* @author dg50
*
*/
public class DeploymentOverview {
private ArrayList<RecordingPeriod> recordingPeriods = new ArrayList<>();
private boolean dutyCycled;
public DeploymentOverview(boolean dutyCycled) {
super();
this.dutyCycled = dutyCycled;
}
public DeploymentOverview(boolean b, ArrayList<RecordingPeriod> tempPeriods) {
this.recordingPeriods = tempPeriods;
}
public void addRecordingPeriod(long start, long stop) {
addRecordingPeriod(new RecordingPeriod(start, stop));
}
private void addRecordingPeriod(RecordingPeriod recordingPeriod) {
recordingPeriods.add(recordingPeriod);
}
public boolean isDutyCycled() {
return dutyCycled;
}
public void setDutyCycled(boolean dutyCycled) {
this.dutyCycled = dutyCycled;
}
public ArrayList<RecordingPeriod> getRecordingPeriods() {
return recordingPeriods;
}
}

11
src/tethys/deployment/DeploymentRecoveryPair.java Normal file
View File

@ -0,0 +1,11 @@
package tethys.output;
import nilus.DeploymentRecoveryDetails;
public class DeploymentRecoveryPair {
public DeploymentRecoveryDetails deploymentDetails;
public DeploymentRecoveryDetails recoveryDetails;
}

5
src/tethys/deployment/DutyCycleInfo.java Normal file
View File

@ -0,0 +1,5 @@
package tethys.deployment;
public class DutyCycleInfo {
}

36
src/tethys/deployment/RecordingPeriod.java Normal file
View File

@ -0,0 +1,36 @@
package tethys.deployment;
public class RecordingPeriod {
private long recordStart;
private long recordStop;
public RecordingPeriod(long recordStart, long recordStop) {
super();
this.recordStart = recordStart;
this.recordStop = recordStop;
}
public long getRecordStart() {
return recordStart;
}
public void setRecordStart(long recordStart) {
this.recordStart = recordStart;
}
public long getRecordStop() {
return recordStop;
}
public void setRecordStop(long recordStop) {
this.recordStop = recordStop;
}
public long getDuration() {
return recordStop-recordStart;
}
}

453
src/tethys/output/TethysExporter.java
View File

@ -1,23 +1,17 @@
package tethys.output;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileAttribute;
import java.io.StringWriter;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import org.apache.commons.beanutils.converters.BigIntegerConverter;
import org.w3c.dom.Document;
import Acquisition.AcquisitionControl;
import Acquisition.AcquisitionParameters;
import Acquisition.AcquisitionProcess;
import Acquisition.DaqStatusDataUnit;
import Acquisition.DaqSystem;
import Array.ArrayManager;
import Array.Hydrophone;
import Array.PamArray;
@ -30,35 +24,32 @@ import PamUtils.PamCalendar;
import PamguardMVC.PamDataBlock;
import PamguardMVC.PamDataUnit;
import PamguardMVC.dataSelector.DataSelector;
import dataMap.OfflineDataMap;
import dataMap.OfflineDataMapPoint;
import generalDatabase.DBControlUnit;
import generalDatabase.DBSchemaWriter;
import generalDatabase.SQLLogging;
import metadata.MetaDataContol;
import metadata.deployment.DeploymentData;
import nilus.Audio;
import nilus.ChannelInfo;
import nilus.ChannelInfo.Sampling;
import nilus.ChannelInfo.Sampling.Regimen;
import nilus.Deployment;
import nilus.Deployment.SamplingDetails;
import nilus.Deployment.Sensors;
import nilus.DeploymentRecoveryDetails;
import pamMaths.PamVector;
import tethys.TethysControl;
import tethys.TethysLocationFuncs;
import tethys.TethysTimeFuncs;
import tethys.dbxml.DBXMLConnect;
import tethys.deployment.DeploymentHandler;
import tethys.deployment.DeploymentRecoveryPair;
import tethys.pamdata.TethysDataProvider;
import tethys.pamdata.TethysSchema;
import dbxml.uploader.Importer;
import nilus.Deployment;
import nilus.MarshalXML;
/**
* Class sitting at the centre of all operations. It will talk to PAMGuard
* objects to get schemas and data and talk to the database connection to move
* data out (and possibly in). Eventually, a lot of the functionality in here
* will be moved to worker threads (SwingWorker?) so that it's easy to keep
* dialogs alive, show progress for big export jobs, etc. For now though, it's a
* relatively simple set of function which we can use to a) open the database,
* b) check everything such as schemas, etc. c) export data and d) clean up.
*
* Class sitting at the centre of all operations. It will talk to PAMGuard objects to get schemas and data
* and talk to the database connection to move data out (and possibly in). Eventually, a lot of the functionality
* in here will be moved to worker threads (SwingWorker?) so that it's easy to keep dialogs alive, show
* progress for big export jobs, etc. For now though, it's a relatively simple set of function which
* we can use to a) open the database, b) check everything such as schemas, etc. c) export data and d) clean up.
* @author dg50
*
*/
@ -66,7 +57,7 @@ public class TethysExporter {
private TethysControl tethysControl;
private TethysExportParams tethysExportParams;
private DBXMLConnect dbxmlConnect;
public TethysExporter(TethysControl tethysControl, TethysExportParams tethysExportParams) {
@ -74,81 +65,50 @@ public class TethysExporter {
this.tethysExportParams = tethysExportParams;
dbxmlConnect = new DBXMLConnect(tethysControl);
}
/**
* Does the work. In reality this will need an awful lot of changing, for
* instance to provide feedback to an observer class to show progress on the
* display.
*
* @return OK if success.
* Does the work. In reality this will need an awful lot of changing, for instance
* to provide feedback to an observer class to show progress on the display.
* @return OK if success.
*/
public boolean doExport() {
// boolean dbOK = dbxmlConnect.openDatabase();
// if (!dbOK) {
/*
* should we set up some exceptions to throw ? Can be a lot more informative
* than a simple 'false'
*/
// return false;
// }
Deployment deployment1 = new Deployment();
deployment1.setId("1");
Path tempFile = null;
try {
JAXBContext jaxB = JAXBContext.newInstance(Deployment.class);
Marshaller marshall = jaxB.createMarshaller();
marshall.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
StringWriter sw = new StringWriter();
marshall.marshal(deployment1, sw);
tempFile = Files.createTempFile("pamGuardToTethys", ".xml");
Files.write(tempFile, sw.toString().getBytes());
String fileText = Importer.ImportFiles("http://localhost:9779", "Deployment",
new String[] { tempFile.toString() }, "", "", false);
tempFile.toFile().deleteOnExit();
} catch(IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (JAXBException e) {
// TODO Auto-generated catch block
e.printStackTrace();
boolean dbOK = dbxmlConnect.openDatabase();
if (!dbOK) {
/*
* should we set up some exceptions to throw ? Can be a lot
* more informative than a simple 'false'
*/
return false;
}
SnapshotGeometry arrayGeometry = findArrayGeometrey();
/**
* Doug populate instrument fields - may need to add a few things. Marie to
* define what we mean by instrument. Instrument names probably need to be added
* to the PAMGuard Array dialog and can then be extraced from there. We had some
* useful discussion about what constitutes an instrumnet in Tinas dataset where
* there was a deployment of 10 MARU's, but the files from all of them were
* merged into a single 10 channel wav file dataset and processed together for
* detection and localisation. Clearly this goes into a single Tethys database,
* but do we put 'MARU Array' as the instrument and then include serial numbers
* of individual MARU's with the hydrophone data, or what ?
* Doug populate instrument fields - may need to add a few things. Marie
* to define what we mean by instrument.
* Instrument names probably need to be added to the PAMGuard Array dialog and can
* then be extraced from there. We had some useful discussion about what constitutes
* an instrumnet in Tinas dataset where there was a deployment of 10 MARU's, but the
* files from all of them were merged into a single 10 channel wav file dataset and
* processed together for detection and localisation. Clearly this goes into a single
* Tethys database, but do we put 'MARU Array' as the instrument and then include
* serial numbers of individual MARU's with the hydrophone data, or what ?
*/
/**
* Doug write something here to get most of the 'samplingdetails' schema. This
* all comes out of the Sound Acquisition module. Code below shows how to find
* this and iterate through various bits of information ... (I've put it in a
* separate function. Currently returning void,but could presumably return a
* Tethys samplingdetails document?)
* Doug write something here to get most of the 'samplingdetails' schema.
* This all comes out of the Sound Acquisition module. Code below shows how
* to find this and iterate through various bits of information ...
* (I've put it in a separate function. Currently returning void,but could
* presumably return a Tethys samplingdetails document?)
*/
/*
* A load of notes Katie put in ....654654654
*/
//1. grab DeploymentRecoveryPair that has deployment details and recovery details
//a. this is based on start and end times
//Douglas calculates out dutycycles to only grab the
@ -210,7 +170,10 @@ public class TethysExporter {
//get list of deployment recovery details (start, stop times and lat/long)
//deployment details and recovery details are same structure
//per pair, go through a loop to fill in each deployment
ArrayList<DeploymentRecoveryPair> deployRecover = getSamplingDetails();
DeploymentHandler deploymentHandler = new DeploymentHandler();
ArrayList<DeploymentRecoveryPair> deployRecover = deploymentHandler.getDeployments();
if (deployRecover == null) {
return false;
}
@ -221,229 +184,111 @@ public class TethysExporter {
int i = 0;
for (DeploymentRecoveryPair drd : deployRecover) {
Deployment deployment = createDeploymentDocument(i++, drd);
Deployment deployment = deploymentHandler.createDeploymentDocument(i++, drd);
}
/*
* Call some general export function
*/
exportGeneralData(tethysExportParams);
/*
* go through the export params and call something for every data block that's
* enabled.
* go through the export params and call something for every
* data block that's enabled.
*/
ArrayList<PamDataBlock> allDataBlocks = PamController.getInstance().getDataBlocks();
for (PamDataBlock aDataBlock : allDataBlocks) {
StreamExportParams streamExportParams = tethysExportParams.getStreamParams(aDataBlock);
if (streamExportParams == null || streamExportParams.selected == false) {
continue; // not interested in this one.
continue; // not interested in this one.
}
exportDataStream(aDataBlock, tethysExportParams, streamExportParams);
}
/*
* Then do whatever else is needed to complete the document.
* Then do whatever else is needed to complete the document.
*/
dbxmlConnect.closeDatabase();
return true;
}
private Deployment createDeploymentDocument(int i, DeploymentRecoveryPair drd) {
Deployment deployment = new Deployment();
deployment.setDeploymentDetails(drd.deploymentDetails);
deployment.setRecoveryDetails(drd.recoveryDetails);
TethysLocationFuncs.getTrackAndPositionData(deployment);
return deployment;
}
/**
* find Deployment data. This is stored in a separate PAMGuard module, which may
* not be present.
*
* @return
* find Deployment data. This is stored in a separate PAMGuard module, which may not
* be present.
* @return
*/
public DeploymentData findDeploymentData() {
/**
* What to do if this isn't present or is incomplete ? Should we be showing this
* in the main export dialog at some point ? More a Q for when we make a nicer
* UI later in the project.
* What to do if this isn't present or is incomplete ? Should we be showing this in the
* main export dialog at some point ? More a Q for when we make a nicer UI later in the project.
*/
MetaDataContol metaControl = (MetaDataContol) PamController.getInstance()
.findControlledUnit(MetaDataContol.unitType);
MetaDataContol metaControl = (MetaDataContol) PamController.getInstance().findControlledUnit(MetaDataContol.unitType);
if (metaControl == null) {
return null;
} else {
}
else {
return metaControl.getDeploymentData();
}
}
public SnapshotGeometry findArrayGeometrey() {
/*
* this should never be null, but again, we might want to put some warnings and
* exception handlers in here anyway. Really just an example to show how to find
* this. We'll need to dig a bit elsewhere to get more detailed hydrophone
* information.
* this should never be null, but again, we might want to put some warnings
* and exception handlers in here anyway. Really just an example to show how
* to find this. We'll need to dig a bit elsewhere to get more detailed
* hydrophone information.
*/
/*
* In PAMGuard hydrophones are assigned to streamers, which can have different
* methods for estimating their positions from GPS. The geometry is a sum of xyz
* in the streamer and xyz in the hydrophone object Within a streamer,
* hydrophones are considered rigid relative to each other. The stremer will
* floow a choice of modesl (rigid, threading, etc) to estimate it's position
* relative to the GPS track. Different errors are used when estimating
* localisation errors within and between streamers. The Snapshot geometry sorts
* a lot of this out for a point in time and will give back a single object
* which is most of what we'll be wanting.
* In PAMGuard hydrophones are assigned to streamers, which can have different methods for estimating
* their positions from GPS. The geometry is a sum of xyz in the streamer and xyz in the hydrophone object
* Within a streamer, hydrophones are considered rigid relative to each other. The stremer will floow
* a choice of modesl (rigid, threading, etc) to estimate it's position relative to the GPS track. Different
* errors are used when estimating localisation errors within and between streamers.
* The Snapshot geometry sorts a lot of this out for a point in time and will give back a single object
* which is most of what we'll be wanting.
*/
PamArray currentArray = ArrayManager.getArrayManager().getCurrentArray();
SnapshotGeometry currentGeometry = currentArray.getSnapshotGeometry(PamCalendar.getTimeInMillis());
/*
* The following lines of code show how to get more detailed calibration info
* for each hydrophone, but we'll have to think about the easiest way to
* repackage this for Tethys. e.g. this function could be modified to return the
* correct Tethys object in one go.
* The following lines of code show how to get more detailed calibration info for each
* hydrophone, but we'll have to think about the easiest way to repackage this for Tethys.
* e.g. this function could be modified to return the correct Tethys object in one go.
*/
ArrayList<Hydrophone> hydrophones = currentArray.getHydrophoneArray();
/*
* each object in the list will have more detailed cal information for each
* phone. But for the full system calibration we'd need to go to the Acquisition
* module.
* each object in the list will have more detailed cal information for each phone. But
* for the full system calibration we'd need to go to the Acquisition module.
*/
AcquisitionControl daqControl = (AcquisitionControl) PamController.getInstance()
.findControlledUnit(AcquisitionControl.unitType);
AcquisitionControl daqControl = (AcquisitionControl) PamController.getInstance().findControlledUnit(AcquisitionControl.unitType);
if (daqControl != null) {
AcquisitionProcess daqProcess = daqControl.getAcquisitionProcess();
for (int iPhone = 0; iPhone < hydrophones.size(); iPhone++) {
Hydrophone aPhone = hydrophones.get(iPhone);
Hydrophone aPhone = hydrophones.get(iPhone);
double totalCal = -daqProcess.rawAmplitude2dB(1, iPhone, false);
System.out.printf(
"hydrophone %d has sensitivity %3.1fdB + gain %3.1fdB. Total calibration is %3.1fdB re1U/uPa\n",
System.out.printf("hydrophone %d has sensitivity %3.1fdB + gain %3.1fdB. Total calibration is %3.1fdB re1U/uPa\n",
iPhone, aPhone.getSensitivity(), aPhone.getPreampGain(), totalCal);
}
}
return currentGeometry;
}
// in each channel
private ArrayList<DeploymentRecoveryPair> getSamplingDetails() {
// first find an acquisition module.
PamControlledUnit aModule = PamController.getInstance().findControlledUnit(AcquisitionControl.class, null);
if (aModule instanceof AcquisitionControl == false) {
// will return if it's null. Impossible for it to be the wrong type.
// but it's good practice to check anyway before casting.
return null;
}
// cast it to the right type.
AcquisitionControl daqControl = (AcquisitionControl) aModule;
AcquisitionParameters daqParams = daqControl.getAcquisitionParameters();
/**
* The daqParams class has most of what we need about the set up in terms of
* sample rate, number of channels, instrument type, ADC input range (part of
* calibration), etc. It also has a hydrophone list, which maps the input
* channel numbers to the hydrophon numbers. Realistically, this list is always
* 0,1,2,etc or it goes horribly wrong !
*/
// so write functions here to get information from the daqParams.
System.out.printf("Sample regime: %s input with rate %3.1fHz, %d channels, gain %3.1fdB, ADCp-p %3.1fV\n",
daqParams.getDaqSystemType(), daqParams.getSampleRate(), daqParams.getNChannels(),
daqParams.preamplifier.getGain(), daqParams.voltsPeak2Peak);
/**
* then there is the actual sampling. This is a bit harder to find. I thought it
* would be in the data map but the datamap is a simple count of what's in the
* databasase which is not quite what we want. we're going to have to query the
* database to get more detailed informatoin I think. I'll do that here for now,
* but we may want to move this when we better organise the code. It also seems
* that there are 'bad' dates in the database when it starts new files, which
* are the date data were analysed at. So we really need to check the start and
* stop records only.
*/
PamDataBlock<DaqStatusDataUnit> daqInfoDataBlock = daqControl.getAcquisitionProcess().getDaqStatusDataBlock();
// just load everything. Probably OK for the acqusition, but will bring down
daqInfoDataBlock.loadViewerData(0, Long.MAX_VALUE, null);
ArrayList<DaqStatusDataUnit> allStatusData = daqInfoDataBlock.getDataCopy();
long dataStart = Long.MAX_VALUE;
long dataEnd = Long.MIN_VALUE;
if (allStatusData != null && allStatusData.size() > 0) {
// find the number of times it started and stopped ....
int nStart = 0, nStop = 0, nFile = 0;
for (DaqStatusDataUnit daqStatus : allStatusData) {
switch (daqStatus.getStatus()) {
case "Start":
nStart++;
dataStart = Math.min(dataStart, daqStatus.getTimeMilliseconds());
break;
case "Stop":
nStop++;
dataEnd = Math.max(dataEnd, daqStatus.getEndTimeInMilliseconds());
break;
case "NextFile":
nFile++;
break;
}
}
System.out.printf(
"Input map of sound data indicates data from %s to %s with %d starts and %d stops over %d files\n",
PamCalendar.formatDateTime(dataStart), PamCalendar.formatDateTime(dataEnd), nStart, nStop,
nFile + 1);
}
// // and we find the datamap within that ...
// OfflineDataMap daqMap = daqInfoDataBlock.getOfflineDataMap(DBControlUnit.findDatabaseControl());
// if (daqMap != null) {
// // iterate through it.
// long dataStart = daqMap.getFirstDataTime();
// long dataEnd = daqMap.getLastDataTime();
// List<OfflineDataMapPoint> mapPoints = daqMap.getMapPoints();
// System.out.printf("Input map of sound data indicates data from %s to %s with %d individual files\n",
// PamCalendar.formatDateTime(dataStart), PamCalendar.formatDateTime(dataEnd), mapPoints.size());
// /*
// * clearly in the first database I've been looking at of Tinas data, this is NOT getting sensible start and
// * end times. Print them out to see what's going on.
// */
//// for ()
// }
DeploymentRecoveryPair pair = new DeploymentRecoveryPair();
DeploymentRecoveryDetails deployment = new DeploymentRecoveryDetails();
DeploymentRecoveryDetails recovery = new DeploymentRecoveryDetails();
pair.deploymentDetails = deployment;
pair.recoveryDetails = recovery;
deployment.setTimeStamp(TethysTimeFuncs.xmlGregCalFromMillis(dataStart));
deployment.setAudioTimeStamp(TethysTimeFuncs.xmlGregCalFromMillis(dataStart));
recovery.setTimeStamp(TethysTimeFuncs.xmlGregCalFromMillis(dataEnd));
recovery.setAudioTimeStamp(TethysTimeFuncs.xmlGregCalFromMillis(dataEnd));
ArrayList<DeploymentRecoveryPair> drPairs = new ArrayList<>();
drPairs.add(pair);
return drPairs;
}
/**
* No idea if we need this or not. May want to return something different to
* void, e.g. a reference to the main object for a tethys export. I've no idea !
*
* No idea if we need this or not. May want to return something different to void, e.g.
* a reference to the main object for a tethys export. I've no idea !
* @param tethysExportParams2
*/
private void exportGeneralData(TethysExportParams tethysExportParams) {
// TODO Auto-generated method stub
}
/**
* Here is where we export data for a specific data stream to Tethys.
* Here is where we export data for a specific data stream to Tethys.
*
* @param aDataBlock
* @param tethysExportParams
@ -452,102 +297,100 @@ public class TethysExporter {
private void exportDataStream(PamDataBlock aDataBlock, TethysExportParams tethysExportParams,
StreamExportParams streamExportParams) {
/**
* This will probably need to be passed additional parameters and may also want
* to return something other than void in order to build a bigger Tethys
* document.
* This will probably need to be passed additional parameters and may also want to return something
* other than void in order to build a bigger Tethys document.
*/
/*
* Some examples of how to do whatever is needed to get schema and data out of
* PAMGuard.
* Some examples of how to do whatever is needed to get schema and data out of PAMGuard.
*/
/*
* first we'll probably want a reference to the module containing the data. in
* principle this can't get null, since the datablock was found be searching in
* the other direction.
* first we'll probably want a reference to the module containing the data.
* in principle this can't get null, since the datablock was found be searching in
* the other direction.
*/
PamControlledUnit pamControlledUnit = aDataBlock.getParentProcess().getPamControlledUnit();
TethysDataProvider dataProvider = aDataBlock.getTethysDataProvider();
PamguardXMLWriter pamXMLWriter = PamguardXMLWriter.getXMLWriter();
if (dataProvider == null) {
return;
}
TethysSchema tethysSchema = dataProvider.getSchema();
/*
* the schema should have a Document object in it. If we wanted to turn that
* into an XML string we can ... (though I'd assume that having the Document is
* more useful)
/*
* the schema should have a Document object in it. If we wanted to turn
* that into an XML string we can ...
* (though I'd assume that having the Document is more useful)
*/
String schemaXMLString = pamXMLWriter.getAsString(tethysSchema.getXsd(), false);
System.out.printf("Schema for %s is %s\n", aDataBlock.getDataName(), schemaXMLString);
/*
* Get the XML settings for that datablock. This is (or should be the parameters
* that were controlling that module, with adequate data about upstream
* modules). I think this has to go somewhere into the Detections document.
* Get the XML settings for that datablock. This is (or should be
* the parameters that were controlling that module, with adequate data about
* upstream modules). I think this has to go somewhere into the Detections document.
*/
Document doc = pamXMLWriter.writeOneModule((PamSettings) pamControlledUnit, System.currentTimeMillis());
String moduleXML = null;
if (doc != null) {
// this string should be XML of all the settings for the module controlling this
// datablock.
// this string should be XML of all the settings for the module controlling this datablock.
moduleXML = pamXMLWriter.getAsString(doc, true); // change to false to get smaller xml
System.out.printf("Module settings for datablock %s are:\n", moduleXML);
System.out.println(moduleXML);
}
/**
* Now can go through the data. Probably, we'll want to go through all the data
* in the project, but we can hold off on that for now and just go for data that
* are in memory. We'll also have to think a lot about updating parts of the
* database which have been reprocessed - what we want to do, should eventually
* all be options set in the dialog and available within TethysExportParams For
* now though, we're just going to export data that are in memory. Once basic
* export is working, I can easily enough write something which will go through
* an entire data set, go through between two times, etc.
* Now can go through the data. Probably, we'll want to go through all the data in
* the project, but we can hold off on that for now and just go for data that
* are in memory. We'll also have to think a lot about updating parts of the
* database which have been reprocessed - what we want to do, should eventually all
* be options set in the dialog and available within TethysExportParams
* For now though, we're just going to export data that are in memory.
* Once basic export is working, I can easily enough write something which will go
* through an entire data set, go through between two times, etc.
*/
// so this is a way of iterating through the data that are in memory, which will
// do for now ..
// do it with a data copy which can avoid synchronising the entire block for
// what may be a long time
// the copy function is itself synched, and is quite fast, so easier and safe
// this way
// so this is a way of iterating through the data that are in memory, which will do for now ..
// do it with a data copy which can avoid synchronising the entire block for what may be a long time
// the copy function is itself synched, and is quite fast, so easier and safe this way
ArrayList<PamDataUnit> dataCopy = aDataBlock.getDataCopy();
DataSelector dataSelector = aDataBlock.getDataSelector(tethysControl.getDataSelectName(), false);
int nSkipped = 0;
int nExport = 0;
for (PamDataUnit aData : dataCopy) {
/*
* see if we want this data unit. PAMGuard has a complicated system of data
* selectors specific to each data type. These are centrally managed so you
* don't need to worry too much about them. They are identified by name for each
* data stream and the behaviour here should follow the selections you made in
* the dialog. the data selection system allows different displays to show
* different data, so a stream can have many differently named selectors active
* at any one time, all doing different things in different parts of PAMGuard.
* see if we want this data unit. PAMGuard has a complicated system of data selectors specific to
* each data type. These are centrally managed so you don't need to worry too much about them. They
* are identified by name for each data stream and the behaviour here should follow the selections you
* made in the dialog.
* the data selection system allows different displays to show different data, so a stream can have many differently
* named selectors active at any one time, all doing different things in different parts of PAMGuard.
*/
if (dataSelector != null) {
if (dataSelector.scoreData(aData) <= 0) {
nSkipped++;
continue; // don't want this one.
continue; // don't want this one.
}
}
/*
* then we do whatever we need to do to convert this into something for Tethys
* this might happen in the tethysSchema object for each data stream ????
* then we do whatever we need to do to convert this into something for Tethys
* this might happen in the tethysSchema object for each data stream ????
*/
nExport++;
nExport ++;
}
System.out.printf("Exported %d data units and skipped %d in %s", nExport, nSkipped,
aDataBlock.getLongDataName());
System.out.printf("Exported %d data units and skipped %d in %s", nExport, nSkipped, aDataBlock.getLongDataName());
}
}