Tidying

Lots of GUI improvement and code tidying. Functionality to export
gzipped documents to reduce traffic.

.classpath

@@ -6,9 +6,8 @@
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER">
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-11">
 		<attributes>
-			<attribute name="module" value="true"/>
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>

dependency-reduced-pom.xml

@@ -4,7 +4,7 @@
   <groupId>org.pamguard</groupId>
   <artifactId>Pamguard</artifactId>
   <name>Pamguard Java12+</name>
-  <version>2.02.10b</version>
+  <version>2.02.10ad</version>
   <description>Pamguard for Java 12+, using Maven to control dependcies</description>
   <url>www.pamguard.org</url>
   <organization>

src/PamUtils/worker/PamWorkDialog.java

@@ -1,5 +1,6 @@
 package PamUtils.worker;
 
+import java.awt.Dimension;
 import java.awt.Window;
 
 import javax.swing.BoxLayout;
@@ -23,6 +24,9 @@ public class PamWorkDialog extends PamDialog {
 		mainPanel.setBorder(new TitledBorder("Task Progress"));
 //		GridBagConstraints c = new PamGridBagContraints();
 		mainPanel.add(progressBar = new JProgressBar(0, 100));
+		Dimension sz = progressBar.getPreferredSize();
+		sz.width = 300;
+		progressBar.setPreferredSize(sz);
 		textRows = new PamTextDisplay[nTextRows];
 		for (int i = 0; i < nTextRows; i++) {
 //			c.gridy++;

src/tethys/CollectionHandler.java

@@ -0,0 +1,38 @@
+package tethys;
+
+abstract public class CollectionHandler {
+
+	private Collection collection;
+	
+	protected TethysControl tethysControl;
+
+	/**
+	 * @param tethysControl
+	 * @param collection
+	 */
+	public CollectionHandler(TethysControl tethysControl, Collection collection) {
+		this.tethysControl = tethysControl;
+		this.collection = collection;
+	}
+	
+	public String collectionName() {
+		return collection.collectionName();
+	}
+
+	/**
+	 * @return the collection
+	 */
+	public Collection getCollection() {
+		return collection;
+	}
+
+	/**
+	 * @return the tethysControl
+	 */
+	public TethysControl getTethysControl() {
+		return tethysControl;
+	}
+	
+	public abstract String getHelpPoint();
+	
+}

src/tethys/TethysControl.java

@@ -469,6 +469,8 @@ public class TethysControl extends PamControlledUnit implements PamSettings, Tet
 //		case PamControllerInterface.INITIALIZATION_COMPLETE:
 			initializationStuff();
 			break;
+		case PamControllerInterface.HYDROPHONE_ARRAY_CHANGED:
+			sendStateUpdate(new TethysState(StateType.UPDATEMETADATA));
 		}
 	}
 

src/tethys/calibration/CalibrationHandler.java

@@ -1,6 +1,5 @@
 package tethys.calibration;
 
-import java.lang.reflect.Field;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -37,6 +36,7 @@ import nilus.MetadataInfo;
 import nilus.QualityValueBasic;
 import nilus.ResponsibleParty;
 import tethys.Collection;
+import tethys.CollectionHandler;
 import tethys.DocumentInfo;
 import tethys.DocumentNilusObject;
 import tethys.TethysControl;
@@ -52,9 +52,7 @@ import tethys.niluswraps.NilusUnpacker;
 import tethys.pamdata.AutoTethysProvider;
 import tethys.reporter.TethysReporter;
 
-public class CalibrationHandler implements TethysStateObserver {
-
-	private TethysControl tethysControl;
+public class CalibrationHandler extends CollectionHandler implements TethysStateObserver {
 	
 	private ArrayList<DocumentNilusObject<Calibration>> calibrationsList;
 	
@@ -65,10 +63,14 @@ public class CalibrationHandler implements TethysStateObserver {
 	public static final String[] qaTypes = {"unverified", "valid", "invalid"};
 	
 	private Helper nilusHelper;
+
+	public static final String helpPoint = "utilities.tethys.docs.calibrations";
+	
 	/**
 	 * @param tethysControl
 	 */
 	public CalibrationHandler(TethysControl tethysControl) {
+		super(tethysControl, Collection.Calibrations);
 		this.tethysControl = tethysControl;
 		calibrationsList = new ArrayList();
 		tethysControl.addStateObserver(this);		try {
@@ -504,7 +506,7 @@ public class CalibrationHandler implements TethysStateObserver {
 		}
 		String seachPattern = makeChannelNamePart(iChan);
 		for (int i = 0; i < calibrationsList.size(); i++) {
-			String docName = calibrationsList.get(i).getDocumentName();
+			String docName = calibrationsList.get(i).getDocumentId();
 			if (docName.endsWith(seachPattern)) {
 				return true;
 			}
@@ -570,4 +572,9 @@ public class CalibrationHandler implements TethysStateObserver {
 		}
 		return theseCals;
 	}
+
+	@Override
+	public String getHelpPoint() {
+		return helpPoint;
+	}
 }

src/tethys/calibration/swing/CalibrationsContactCard.java

@@ -5,6 +5,7 @@ import java.awt.GridBagConstraints;
 import java.awt.GridBagLayout;
 import java.awt.event.ActionEvent;
 import java.awt.event.ActionListener;
+import java.util.ArrayList;
 import java.util.Date;
 
 import javax.swing.BoxLayout;
@@ -18,6 +19,9 @@ import javax.xml.datatype.XMLGregorianCalendar;
 
 import org.jdesktop.swingx.JXDatePicker;
 
+import Array.ArrayManager;
+import Array.Hydrophone;
+import Array.PamArray;
 import PamView.dialog.PamDialog;
 import PamView.dialog.PamGridBagContraints;
 import PamView.panel.WestAlignedPanel;
@@ -43,6 +47,8 @@ public class CalibrationsContactCard extends CalibrationsCard {
 	
 	private JButton copyDown, copyUp;
 	
+	private JComboBox<String> hydrophoneSelection;
+	
 	public CalibrationsContactCard(PamWizard pamWizard) {
 		super(pamWizard, "Contact Details");
 		// TODO Auto-generated constructor stub
@@ -57,17 +63,26 @@ public class CalibrationsContactCard extends CalibrationsCard {
 		
 		JPanel datePanel = new JPanel(new GridBagLayout());
 		JPanel lp = new WestAlignedPanel(datePanel);
-		lp.setBorder(new TitledBorder("Calibration date"));
+		lp.setBorder(new TitledBorder("Date and hydrophones"));
 		GridBagConstraints c = new PamGridBagContraints();
 		datePanel.add(new JLabel("Calibration date: ", JLabel.RIGHT), c);
 		datePicker = new JXDatePicker();
 		c.gridx++;
 		datePanel.add(datePicker, c);
-		c.gridx = 0;
-		c.gridy++;
-		datePanel.add(new JLabel("Update Frequency", JLabel.RIGHT), c);
+//		c.gridx = 0;
+		c.gridx++;
+		datePanel.add(new JLabel("  Update Frequency ", JLabel.RIGHT), c);
 		c.gridx++;
 		datePanel.add(updateInterval, c);
+		c.gridx = 0;
+		c.gridy++;
+		datePanel.add(new JLabel("  Hydrophones ", JLabel.RIGHT), c);
+		c.gridwidth = 5;
+		c.gridx++;
+		hydrophoneSelection = new JComboBox<>();
+		datePanel.add(hydrophoneSelection, c);
+		hydrophoneSelection.setToolTipText("Select which hydrophone calibrations to export");
+		
 		
 		calibrator = new ResponsiblePartyPanel("Technical Person");
 		dataManager = new ResponsiblePartyPanel("Data Manager");
@@ -202,6 +217,16 @@ public class CalibrationsContactCard extends CalibrationsCard {
 			datePicker.setDate(new Date(TethysTimeFuncs.millisFromGregorianXML(ts)));
 		}
 		
+		hydrophoneSelection.removeAllItems();
+		hydrophoneSelection.addItem("All hydrophones");
+		PamArray array = ArrayManager.getArrayManager().getCurrentArray();
+		ArrayList<Hydrophone> phones = array.getHydrophoneArray();
+		int i = 0;
+		for (Hydrophone phone : phones) {
+			String txt = String.format("Hydrophone %d, %s, %3.1f dBre1\u00B5Pa", i, phone.getType(), phone.getSensitivity());
+			hydrophoneSelection.addItem(txt);
+			i++;
+		}
 
 	}
 

src/tethys/calibration/swing/CalibrationsExportWizard.java

@@ -5,6 +5,7 @@ import java.awt.Window;
 import PamView.wizard.PamWizard;
 import PamView.wizard.PamWizardCard;
 import nilus.Calibration;
+import tethys.calibration.CalibrationHandler;
 
 public class CalibrationsExportWizard extends PamWizard {
 
@@ -13,8 +14,9 @@ public class CalibrationsExportWizard extends PamWizard {
 	private CalibrationsExportWizard(Window parentFrame, Calibration sampleDocument) {
 		super(parentFrame, "Calibrations Export");
 		this.sampleDocument = sampleDocument;
-		addCard(new CalibrationProcessCard(this));
 		addCard(new CalibrationsContactCard(this));
+		addCard(new CalibrationProcessCard(this));
+		setHelpPoint(CalibrationHandler.helpPoint);
 	}
 	
 	public static Calibration showWizard(Window parentFrame, Calibration sampleDocument) {

src/tethys/calibration/swing/CalibrationsMainPanel.java

@@ -15,53 +15,47 @@ import PamView.panel.PamPanel;
 import tethys.TethysControl;
 import tethys.TethysState;
 import tethys.calibration.CalibrationHandler;
+import tethys.deployment.PInstrument;
+import tethys.swing.TethysExportPanel;
 import tethys.swing.TethysGUIPanel;
+import tethys.swing.TippedButton;
 
-public class CalibrationsMainPanel extends TethysGUIPanel {
+public class CalibrationsMainPanel extends TethysExportPanel {
 
 	private CalibrationHandler calibrationHandler;
 	
 	private CalibrationsTable calibrationsTable;
 	
-	private JPanel mainPanel;
-	
-	private JPanel ctrlPanel;
-	
-	private JButton exportButton;
-	
-	private JLabel warning;
+//	private JPanel mainPanel;
+//	
+//	private JPanel ctrlPanel;
+//	
+//	private TippedButton exportButton;
+//	
+//	private JLabel warning;
 
 	public CalibrationsMainPanel(TethysControl tethysControl, CalibrationHandler calibrationHandler) {
-		super(tethysControl);
+		super(tethysControl, calibrationHandler, false);
 		this.calibrationHandler = calibrationHandler;
-		mainPanel = new PamPanel(new BorderLayout());
+		JPanel mainPanel = getMainPanel();
+//		mainPanel = new PamPanel(new BorderLayout());
 		mainPanel.setBorder(new TitledBorder("Instrument calibration information"));
 		
 		calibrationsTable = new CalibrationsTable(tethysControl, calibrationHandler);
 		mainPanel.add(BorderLayout.CENTER, calibrationsTable.getComponent());
 		
-		ctrlPanel = new PamPanel(new BorderLayout());
-		exportButton = new JButton("Export ...");
-		ctrlPanel.add(BorderLayout.WEST, exportButton);
-		warning = new JLabel();
-		ctrlPanel.add(BorderLayout.CENTER, warning);
-		mainPanel.add(BorderLayout.NORTH, ctrlPanel);
-		exportButton.setToolTipText("Export calibration data to database");
-		exportButton.addActionListener(new ActionListener() {
-			@Override
-			public void actionPerformed(ActionEvent e) {
-				exportCalibrations();
-			}
-		});
-	}
-
-	protected void exportCalibrations() {
-		calibrationHandler.exportAllCalibrations();
-	}
-
-	@Override
-	public JComponent getComponent() {
-		return mainPanel;
+//		ctrlPanel = new PamPanel(new BorderLayout());
+//		exportButton = new TippedButton("Export ...", "Export calibration data to database");
+//		ctrlPanel.add(BorderLayout.WEST, exportButton);
+//		warning = new JLabel();
+//		ctrlPanel.add(BorderLayout.CENTER, warning);
+//		mainPanel.add(BorderLayout.NORTH, ctrlPanel);
+//		exportButton.addActionListener(new ActionListener() {
+//			@Override
+//			public void actionPerformed(ActionEvent e) {
+//				exportCalibrations();
+//			}
+//		});
 	}
 
 	@Override
@@ -71,7 +65,45 @@ public class CalibrationsMainPanel extends TethysGUIPanel {
 	}
 
 	private void enableControls() {
-		exportButton.setEnabled(getTethysControl().isServerOk());
+		if (getTethysControl().isServerOk() == false) {
+			disableExport("Tethys Server not running");
+			return;
+		}
+		if (isHydrophoneNamed() == false) {
+			disableExport("Can't export calibrations until the Hydrophone array has been correctly named");
+			return;
+		};
+		enableExport(true);
+	}
+
+	/**
+	 * Check to see if hydrophone is named correctly. 
+	 * @return
+	 */
+	private boolean isHydrophoneNamed() {
+		PInstrument currentInstrument = getTethysControl().getDeploymentHandler().getCurrentArrayInstrument();
+		if (currentInstrument == null) {
+			return false;
+		}
+		if (currentInstrument.instrumentId == null || currentInstrument.instrumentType == null) {
+			return false;
+		}
+
+		if (currentInstrument.instrumentId.length() == 0 || currentInstrument.instrumentType.length() == 0) {
+			return false;
+		}
+		return true;
+	}
+
+	@Override
+	protected void exportButtonPressed(ActionEvent e) {
+		calibrationHandler.exportAllCalibrations();
+	}
+
+	@Override
+	protected void optionsButtonPressed(ActionEvent e) {
+		// TODO Auto-generated method stub
+		
 	}
 
 }

src/tethys/dbxml/DBXMLConnect.java

@@ -3,14 +3,17 @@ package tethys.dbxml;
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.io.InputStream;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
-import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.zip.GZIPOutputStream;
 
 import javax.xml.bind.JAXBException;
 
@@ -192,12 +195,35 @@ public class DBXMLConnect {
 			MarshalXML marshal = new MarshalXML();
 			marshal.createInstance(objClass);
 			marshal.marshal(nilusObject, tempFile.toString());
+			// above lines have made a file. Are now going to gzip it before sending to Tethys
+			File zipFile = null;
+			try {
+				zipFile = zipOutputFile(tempFile);
+			}
+			catch (FileNotFoundException e1){
+				System.out.println(e1.getMessage());
+			}
+			catch (IOException e2) {
+				System.out.println(e2.getMessage());
+			}
+			String finalName;
+			if (zipFile == null) {
+				finalName = bodgeName;
+			}
+			else {
+				finalName = zipFile.toString();
+			}
+
+
 			//				tempFile = stripXMLHeader(tempFile);
 			importReturn = Importer.ImportFiles(params.getFullServerName(), collection.collectionName(),
-					new String[] { bodgeName }, "", "", false);
+					new String[] { finalName }, "", "", false);
 
 
 			tempFile.deleteOnExit();
+			if (zipFile != null) {
+				zipFile.deleteOnExit();
+			}
 		} catch(IllegalArgumentException e) {
 			throw new TethysException("IllegalArgumentException posting to Tethys: " + e.getMessage(), null);
 		} catch (IOException e) {
@@ -222,6 +248,34 @@ public class DBXMLConnect {
 		return success;
 	}
 
+	/**
+	 * Zip an xml file (or any file) into a gz file with a new end
+	 * @param xmlFile
+	 * @return
+	 * @throws FileNotFoundException
+	 * @throws IOException
+	 */
+	private File zipOutputFile(File xmlFile) throws FileNotFoundException, IOException {
+		String zipName = xmlFile.toString() + "-temp-.gz";
+		File zipFile = new File(zipName);
+		GZIPOutputStream opStream = new GZIPOutputStream(new FileOutputStream(zipFile));
+		InputStream fis = new FileInputStream(xmlFile);
+		int chunkSize = 100*1024;
+		byte[] buffer = new byte[chunkSize];
+//		ZipEntry zipEntry = new ZipEntry(xmlFile.getName());
+		int bytesRead;
+		while ((bytesRead = fis.read(buffer)) >= 0) {
+			opStream.write(buffer, 0, bytesRead);
+		}
+
+		opStream.close();
+
+		fis.close();
+
+
+		return zipFile;
+	}
+
 	/**
 	 * Update a document within Tethys. We're assuming that a
 	 * document with the same name in the same collection already

src/tethys/dbxml/DBXMLQueries.java

@@ -515,6 +515,9 @@ public class DBXMLQueries {
 		 * first query for Detections documents associated with this deployment and datablock.
 		 * updated May 23
 		 */
+		if (dataBlock == null) {
+			return null;
+		}
 		String queryNoDepl = "{\"species\":{\"query\":{\"op\":\"lib:completename2tsn\",\"optype\":\"function\",\"operands\":[\"%s\"]},\"return\":{\"op\":\"lib:tsn2completename\",\"optype\":\"function\",\"operands\":[\"%s\"]}},\"return\":[\"Detections/Id\"],\"select\":[{\"op\":\"=\",\"operands\":[\"Detections/Algorithm/Software\",\"LongDataName\"],\"optype\":\"binary\"}],\"enclose\":1}";
 		String queryWithDepl = "{\"species\":{\"query\":{\"op\":\"lib:completename2tsn\",\"optype\":\"function\",\"operands\":[\"%s\"]},\"return\":{\"op\":\"lib:tsn2completename\",\"optype\":\"function\",\"operands\":[\"%s\"]}},\"return\":[\"Detections/Id\"],\"select\":[{\"op\":\"=\",\"operands\":[\"Detections/DataSource/DeploymentId\",\"TheDeploymentId\"],\"optype\":\"binary\"},{\"op\":\"=\",\"operands\":[\"Detections/Algorithm/Software\",\"LongDataName\"],\"optype\":\"binary\"}],\"enclose\":1}";
 		String query;

src/tethys/deployment/DeploymentHandler.java

@@ -73,6 +73,7 @@ import nilus.UnknownSensor;
 import pamMaths.PamVector;
 import pamMaths.STD;
 import tethys.Collection;
+import tethys.CollectionHandler;
 import tethys.TethysControl;
 import tethys.TethysLocationFuncs;
 import tethys.TethysState;
@@ -83,6 +84,7 @@ import tethys.TethysState.StateType;
 import tethys.dbxml.DBXMLConnect;
 import tethys.dbxml.TethysException;
 import tethys.deployment.swing.DeploymentWizard;
+import tethys.deployment.swing.EffortProblemDialog;
 import tethys.deployment.swing.RecordingGapDialog;
 import tethys.niluswraps.PDeployment;
 import tethys.output.TethysExportParams;
@@ -98,16 +100,9 @@ import tethys.swing.DeploymentTableObserver;
  * @author dg50
  *
  */
-public class DeploymentHandler implements TethysStateObserver, DeploymentTableObserver {
+public class DeploymentHandler extends CollectionHandler implements TethysStateObserver, DeploymentTableObserver {
 	
-	private TethysControl tethysControl;
-	
-	/**
-	 * @return the tethysControl
-	 */
-	public TethysControl getTethysControl() {
-		return tethysControl;
-	}
+//	private TethysControl tethysControl;
 	
 	private EffortFunctions effortFunctions;
 
@@ -119,8 +114,11 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 	
 	private DeploymentExportOpts deploymentExportOptions = new DeploymentExportOpts();
 
+	public static final String helpPoint = "utilities.tethys.docs.deployments";
+
 	public DeploymentHandler(TethysControl tethysControl) {
-		super();
+		
+		super(tethysControl, Collection.Deployments);
 		
 		this.tethysControl = tethysControl;
 		
@@ -368,8 +366,39 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 
 	public void createPamguardOverview() {
 		deploymentOverview = effortFunctions.makeRecordingOverview();
+		
+		checkDeploymentOverview(deploymentOverview);
+		
 		updateProjectDeployments();
 		matchPamguard2Tethys(deploymentOverview, projectDeployments);
+		
+		tethysControl.sendStateUpdate(new TethysState(StateType.NEWPAMGUARDSELECTION));
+	}
+
+	/**
+	 * Check the deployment overview for consistency.<br>
+	 * Take the raw audio information and the binary information and check they are similar.
+	 * if not, ask the user what to do. 
+	 * @param deploymentOverview
+	 */
+	private void checkDeploymentOverview(DeploymentOverview overview) {
+		RecordingList rawList = overview.getRawDataList();
+		RecordingList binList = overview.getBinaryDataList();
+		if (rawList == null || binList == null) {
+			return; // nothing to do
+		}
+		double similarity = rawList.getSimilarity(binList);
+		if (similarity > 0.95) {
+			return;
+		}
+		/*
+		 *  if we get here, it seems like the two lists are very different, so
+		 *  show a dialog to ask the user what to do. 
+		 */
+		RecordingList selList = EffortProblemDialog.showDialog(tethysControl.getGuiFrame(), overview);
+		if (selList != null) {
+			tethysControl.getTethysExportParams().setEffortSourceName(selList.getSourceName());
+		}
 	}
 
 	/**
@@ -381,22 +410,22 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 		if (exportOptions != null) {
 			this.deploymentExportOptions = exportOptions;
 			deploymentOverview = getDeploymentOverview();
-			ArrayList<RecordingPeriod> allPeriods = deploymentOverview.getRecordingPeriods();
+			RecordingList allPeriods = deploymentOverview.getMasterList(getTethysControl());
 			exportDeployments(allPeriods);
 		}
 	}
 
 	/**
 	 * Export deployments docs. Playing with a couple of different ways of doing this. 
-	 * @param selectedDeployments
+	 * @param allPeriods
 	 */
-	public void exportDeployments(ArrayList<RecordingPeriod> selectedDeployments) {
+	public void exportDeployments(RecordingList allPeriods) {
 		TethysReporter.getTethysReporter().clear();
 		if (deploymentExportOptions.separateDeployments) {
-			exportSeparateDeployments(selectedDeployments);
+			exportSeparateDeployments(allPeriods);
 		}
 		else {
-			exportOneDeploymnet(selectedDeployments);
+			exportOneDeploymnet(allPeriods);
 		}
 		TethysReporter.getTethysReporter().showReport(tethysControl.getGuiFrame(), true);
 	}
@@ -404,7 +433,7 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 	/**
 	 * Make one big deployment document with all the recording periods in it. 
 	 */
-	private void exportOneDeploymnet(ArrayList<RecordingPeriod> selectedDeployments) {
+	private void exportOneDeploymnet(RecordingList recordingList) {
 		// do the lot, whatever ...
 		Float sampleRate = null;
 		AcquisitionControl daq = (AcquisitionControl) PamController.getInstance().findControlledUnit(AcquisitionControl.class, null);
@@ -414,10 +443,9 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 			sampleRate = daqParams.sampleRate;
 		}
 		
-		selectedDeployments = getDeploymentOverview().getRecordingPeriods();
 		int freeId = getTethysControl().getDeploymentHandler().getFirstFreeDeploymentId();
-		RecordingPeriod onePeriod = new RecordingPeriod(selectedDeployments.get(0).getRecordStart(), 
-				selectedDeployments.get(selectedDeployments.size()-1).getRecordStop());
+		RecordingPeriod onePeriod = new RecordingPeriod(recordingList.getStart(), 
+				recordingList.getEnd());
 		TethysExportParams exportParams = tethysControl.getTethysExportParams();
 		String id = String.format("%s_%s", exportParams.getDatasetName(), "all");
 		Deployment deployment = createDeploymentDocument(freeId, onePeriod, id);
@@ -425,7 +453,8 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 		Deployment globalMeta = getTethysControl().getGlobalDeplopymentData();
 		deployment.setCruise(globalMeta.getCruise());
 		deployment.setSite(globalMeta.getSite());
-		if (selectedDeployments.size() > 1) {
+		ArrayList<RecordingPeriod> effortPeriods = recordingList.getEffortPeriods();
+		if (recordingList.size() > 1) {
 //			// now need to remove the sampling details - don't though, add invalid periods instead. 
 //			SamplingDetails samplingDetails = deployment.getSamplingDetails();
 //			samplingDetails.getChannel().clear();
@@ -440,9 +469,9 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 				deployment.setQualityAssurance(qa = new AcousticDataQAType());
 			}
 			List<Quality> qualityList = qa.getQuality();
-			for (int i = 1; i < selectedDeployments.size(); i++) {
-				long end = selectedDeployments.get(i-1).getRecordStop();
-				long start = selectedDeployments.get(i).getRecordStart();
+			for (int i = 1; i < recordingList.size(); i++) {
+				long end = effortPeriods.get(i-1).getRecordStop();
+				long start = effortPeriods.get(i).getRecordStart();
 				Quality q = new Quality();
 				q.setStart(TethysTimeFuncs.xmlGregCalFromMillis(end));
 				q.setEnd(TethysTimeFuncs.xmlGregCalFromMillis(start));
@@ -479,14 +508,15 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 	/**
 	 * Make a separate deployment document for every recording period. 
 	 */
-	private void exportSeparateDeployments(ArrayList<RecordingPeriod> selectedDeployments) {
+	private void exportSeparateDeployments(RecordingList recordingList) {
 		
 		int freeId = getTethysControl().getDeploymentHandler().getFirstFreeDeploymentId();
 		// fill in a few things from here
 		Deployment globalMeta = getTethysControl().getGlobalDeplopymentData();
 		TethysExportParams exportParams = tethysControl.getTethysExportParams();
-		for (int i = 0; i < selectedDeployments.size(); i++) {
-			RecordingPeriod recordPeriod = selectedDeployments.get(i);
+		ArrayList<RecordingPeriod> effortPeriods = recordingList.getEffortPeriods();
+		for (int i = 0; i < recordingList.size(); i++) {
+			RecordingPeriod recordPeriod = effortPeriods.get(i);
 			PDeployment exDeploymnet = recordPeriod.getMatchedTethysDeployment();
 			Deployment deployment = null;
 			String id = String.format("%s_%d", exportParams.getDatasetName(), i);
@@ -532,8 +562,9 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 		if (deployments == null || deploymentOverview == null) {
 			return;
 		}
-		ArrayList<RecordingPeriod> recordingPeriods = deploymentOverview.getRecordingPeriods();
-		for (RecordingPeriod aPeriod : recordingPeriods) {
+		RecordingList recordingList = deploymentOverview.getMasterList(getTethysControl());
+		ArrayList<RecordingPeriod> effortPeriods = recordingList.getEffortPeriods();
+		for (RecordingPeriod aPeriod : effortPeriods) {
 			PDeployment closestDeployment = findClosestDeployment(aPeriod, deployments);
 			aPeriod.setMatchedTethysDeployment(closestDeployment);
 			if (closestDeployment != null) {
@@ -592,7 +623,8 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 		if (deploymentOverview == null) {
 			return matched;
 		}
-		for (RecordingPeriod period : deploymentOverview.getRecordingPeriods()) {
+		ArrayList<RecordingPeriod> effortPeriods = deploymentOverview.getMasterList(getTethysControl()).getEffortPeriods();
+		for (RecordingPeriod period : effortPeriods) {
 			PDeployment deployment = period.getMatchedTethysDeployment();
 			if (deployment != null) {
 				if (matched.contains(deployment) == false) {
@@ -1230,7 +1262,8 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 			}
 			regimens.add(regimen);
 			
-			DutyCycleInfo dutyCycleInf = deploymentOverview.getDutyCycleInfo();
+			RecordingList recordingList = deploymentOverview.getMasterList(getTethysControl());
+			DutyCycleInfo dutyCycleInf = recordingList.assessDutyCycle();
 			boolean isDS = dutyCycleInf != null && dutyCycleInf.isDutyCycled;
 			if (isDS) {
 				DutyCycle dutyCycle = new DutyCycle();
@@ -1307,4 +1340,9 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 		return deploymentExportOptions;
 	}
 
+	@Override
+	public String getHelpPoint() {
+		return helpPoint;
+	}
+
 }

src/tethys/deployment/DeploymentOverview.java

@@ -1,16 +1,6 @@
 package tethys.deployment;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.ListIterator;
-
-import Acquisition.AcquisitionControl;
-import Acquisition.AcquisitionParameters;
-import Acquisition.DaqStatusDataUnit;
-import PamController.PamControlledUnit;
-import PamController.PamController;
-import PamguardMVC.PamDataBlock;
+import tethys.TethysControl;
 
 /**
  * Class to give a general overview of all the effort in PAMGuard which will form the
@@ -22,59 +12,83 @@ import PamguardMVC.PamDataBlock;
  */
 public class DeploymentOverview {
 	
-	private ArrayList<RecordingPeriod> recordingPeriods = new ArrayList<>();
+	private RecordingList rawDataList;
 	
-	private DutyCycleInfo dutyCycleInfo;
+	private RecordingList binaryDataList;
 	
-	public DeploymentOverview(DutyCycleInfo dutyCycleInfo) {
-		super();
-		this.dutyCycleInfo = dutyCycleInfo;
-	}
+//	private DutyCycleInfo dutyCycleInfo;
 
-	public DeploymentOverview(DutyCycleInfo dutyCycleInfo, ArrayList<RecordingPeriod> tempPeriods) {
-		this.dutyCycleInfo = dutyCycleInfo;
-		this.recordingPeriods = tempPeriods;
-	}
+//	public DeploymentOverview(DutyCycleInfo dutyCycleInfo) {
+//		super();
+//		this.dutyCycleInfo = dutyCycleInfo;
+//	}
 	
-	
-	public void addRecordingPeriod(long start, long stop) {
-		addRecordingPeriod(new RecordingPeriod(start, stop));
-	}
-
-	private void addRecordingPeriod(RecordingPeriod recordingPeriod) {
-		recordingPeriods.add(recordingPeriod);
-	}
-
-	public ArrayList<RecordingPeriod> getRecordingPeriods() {
-		return recordingPeriods;
-	}
-
-	public DutyCycleInfo getDutyCycleInfo() {
-		return dutyCycleInfo;
+	public DeploymentOverview(DutyCycleInfo dutyCycleInfo, RecordingList rawDataList, RecordingList binaryDataList) {
+//		this.dutyCycleInfo = dutyCycleInfo;
+		this.rawDataList = rawDataList;
+		this.binaryDataList = binaryDataList;
 	}
 
 	/**
-	 * Get the start time of the first recording
-	 * @return
+	 * @return the rawDataList
 	 */
-	public Long getFirstStart() {
-		if (recordingPeriods.size() > 0) {
-			return recordingPeriods.get(0).getRecordStart();
-		}
-		return null;
+	public RecordingList getRawDataList() {
+		return rawDataList;
 	}
 
 	/**
-	 * Get the end time of the last recording
-	 * @return
+	 * @return the binaryDataList
 	 */
-	public Long getLastEnd() {
-		if (recordingPeriods.size() > 0) {
-			return recordingPeriods.get(recordingPeriods.size()-1).getRecordStop();
-		}
-		return null;
+	public RecordingList getBinaryDataList() {
+		return binaryDataList;
 	}
 
+//	/**
+//	 * @return the dutyCycleInfo
+//	 */
+//	public DutyCycleInfo getDutyCycleInfo() {
+//		return dutyCycleInfo;
+//	}
+	
+	public RecordingList getMasterList(TethysControl tethysControl) {
+		return getMasterList(tethysControl.getTethysExportParams().getEffortSourceName());
+	}
+
+	public RecordingList getMasterList(String effortSourceName) {
+		if (effortSourceName == null) {
+			return getLongestList();
+		}
+		if (binaryDataList != null & binaryDataList.getSourceName().equals(effortSourceName)) {
+			return binaryDataList;
+		}
+		if (rawDataList != null & rawDataList.getSourceName().equals(effortSourceName)) {
+			return rawDataList;
+		}
+		return getLongestList();
+	}
+	
+	/**
+	 * Get the recording list with the greatest duration (start to end)
+	 * not looking at coverage between those times. 
+	 * @return
+	 */
+	public RecordingList getLongestList() {
+		if (binaryDataList == null) {
+			return rawDataList;
+		}
+		if (rawDataList == null) {
+			return binaryDataList;
+		}
+		long lRaw = rawDataList.duration();
+		long lBin = binaryDataList.duration();
+		
+		if (lRaw > lBin) {
+			return rawDataList;
+		}
+		else {
+			return binaryDataList;
+		}
+	}
 
 
 }

src/tethys/deployment/EffortFunctions.java

@@ -41,36 +41,40 @@ public class EffortFunctions {
 		this.tethysControl = tethysControl;
 	}
 
-	private DeploymentOverview createOverview(RecordingList tempPeriods) {
-
-		DutyCycleInfo dutyCycleinfo = assessDutyCycle(tempPeriods);
-		if (dutyCycleinfo == null) {
-			return null;
-		}
-
-		// if it's duty cycles, then we only want a single entry. 
-		RecordingList deploymentPeriods;
-		if (dutyCycleinfo.isDutyCycled == false) {
-			deploymentPeriods = tempPeriods;
-		}
-		else {
-			deploymentPeriods = new RecordingList();
-			deploymentPeriods.add(new RecordingPeriod(tempPeriods.get(0).getRecordStart(), tempPeriods.get(tempPeriods.size()-1).getRecordStop()));
-		}
-		/*
-		 * do another sort of the deploymentPeriods. The start stops were in the order they went into the 
-		 * database in the hope that pairs were the right way round. Now check all data are/
-		 */
-		Collections.sort(deploymentPeriods, new Comparator<RecordingPeriod>() {
-			@Override
-			public int compare(RecordingPeriod o1, RecordingPeriod o2) {
-				return (int) (o1.getRecordStart()-o2.getRecordStart());
-			}
-		});
-
-		DeploymentOverview deploymentOverview = new DeploymentOverview(dutyCycleinfo, deploymentPeriods);
-		return deploymentOverview;
-	}
+//	private DeploymentOverview createOverview(RecordingList tempPeriods) {
+//
+//		tempPeriods.sort();
+//		
+//		DutyCycleInfo dutyCycleinfo = assessDutyCycle(tempPeriods);
+//		if (dutyCycleinfo == null) {
+//			return null;
+//		}
+//		
+//
+//		// if it's duty cycles, then we only want a single entry. 
+//		RecordingList deploymentPeriods;
+//		if (dutyCycleinfo.isDutyCycled == false) {
+//			deploymentPeriods = tempPeriods;
+//		}
+//		else {
+//			deploymentPeriods = new RecordingList(tempPeriods.getSourceName());
+//			deploymentPeriods.add(new RecordingPeriod(tempPeriods.get(0).getRecordStart(), tempPeriods.get(tempPeriods.size()-1).getRecordStop()));
+//		}
+//		/*
+//		 * do another sort of the deploymentPeriods. The start stops were in the order they went into the 
+//		 * database in the hope that pairs were the right way round. Now check all data are/
+//		 */
+//		deploymentPeriods.sort();
+////		Collections.sort(deploymentPeriods, new Comparator<RecordingPeriod>() {
+////			@Override
+////			public int compare(RecordingPeriod o1, RecordingPeriod o2) {
+////				return (int) (o1.getRecordStart()-o2.getRecordStart());
+////			}
+////		});
+//
+//		DeploymentOverview deploymentOverview = new DeploymentOverview(dutyCycleinfo, deploymentPeriods);
+//		return deploymentOverview;
+//	}
 
 
 	public DeploymentOverview makeRecordingOverview() {
@@ -79,13 +83,18 @@ public class EffortFunctions {
 		
 		RecordingList binaryPeriods = listBinaryFiles();
 		
-		long l1 = listDuration(recordingPeriods);
-		long l2 = listDuration(binaryPeriods);
-		if (listDuration(binaryPeriods) > listDuration(recordingPeriods)) {
-			recordingPeriods = binaryPeriods;
-		}
+		// see what the similarity is between them
+//		double sim = recordingPeriods.getSimilarity(binaryPeriods);
+//		double testSim = recordingPeriods.getSimilarity(recordingPeriods);
 		
-		DeploymentOverview deploymentOverview = createOverview(recordingPeriods);
+//		long l1 = listDuration(recordingPeriods);
+//		long l2 = listDuration(binaryPeriods);
+//		if (listDuration(binaryPeriods) > listDuration(recordingPeriods)) {
+//			recordingPeriods = binaryPeriods;
+//		}
+//		
+//		DeploymentOverview deploymentOverview = createOverview(recordingPeriods);
+		DeploymentOverview deploymentOverview = new DeploymentOverview(null, recordingPeriods, binaryPeriods);
 		
 		return deploymentOverview;
 	}
@@ -128,7 +137,8 @@ public class EffortFunctions {
 				}
 			}
 		}
-		bestList = mergeRecordings(bestList);
+		DeploymentExportOpts exportOptions = tethysControl.getDeploymentHandler().getDeploymentExportOptions();
+		bestList.mergeRecordingPeriods(exportOptions.maxRecordingGapSeconds*1000);
 		return bestList;
 	}
 	
@@ -138,7 +148,7 @@ public class EffortFunctions {
 		if (mapPoints == null) {
 			return null;
 		}
-		RecordingList periods = new RecordingList();
+		RecordingList periods = new RecordingList(dataMap.getDataMapName());
 		for (OfflineDataMapPoint mapPoint : mapPoints) {
 			periods.add(new RecordingPeriod(mapPoint.getStartTime(), mapPoint.getEndTime()));
 		}
@@ -219,117 +229,60 @@ public class EffortFunctions {
 		//							PamCalendar.formatDBDateTime(aP.getRecordStop()));
 		//				}
 
-		tempPeriods = mergeRecordings(tempPeriods);
-
-		return tempPeriods;
-	}
-
-	/**
-	 * Merge close recordings and discard ones that are too short. 
-	 * @param tempPeriods all recording periods, may be from consecutive files. 
-	 * @return merged list. 
-	 */
-	private RecordingList mergeRecordings(RecordingList tempPeriods) {
-		// now go through those and merge into longer periods where there is no gap between files.
-		if (tempPeriods == null) {
-			return null;
-		}
-
+		tempPeriods.sort();
 		DeploymentExportOpts exportOptions = tethysControl.getDeploymentHandler().getDeploymentExportOptions();
-
-		ListIterator<RecordingPeriod> iterator = tempPeriods.listIterator();
-		RecordingPeriod prevPeriod = null;
-		while (iterator.hasNext()) {
-			RecordingPeriod nextPeriod = iterator.next();
-			long nextDur = nextPeriod.getRecordStop()-nextPeriod.getRecordStart();
-			if (nextDur == 0) {
-				continue;
-			}
-			if (prevPeriod != null) {
-				long gap = nextPeriod.getRecordStart() - prevPeriod.getRecordStop();
-				long prevDur = prevPeriod.getRecordStop()-prevPeriod.getRecordStart();
-				if (gap < exportOptions.maxRecordingGapSeconds*1000) {
-					// ignoring up to 3s gap or a sample error < 2%.Dunno if this is sensible or not.
-					prevPeriod.setRecordStop(nextPeriod.getRecordStop());
-					iterator.remove();
-					nextPeriod = prevPeriod;
-				}
-			}
-			prevPeriod = nextPeriod;
-		}
-		// now remove ones which are too short even after merging. 
-		iterator = tempPeriods.listIterator();
-		while (iterator.hasNext()) {
-			RecordingPeriod nextPeriod = iterator.next();
-			long duration = nextPeriod.getDuration();
-			if (duration < exportOptions.minRecordingLengthSeconds*1000L) {
-				iterator.remove();
-			}
-		}
+		tempPeriods.mergeRecordingPeriods(exportOptions.maxRecordingGapSeconds*1000);
 
 		return tempPeriods;
 	}
 
-	/**
-	 * Work out whether or not the data are evenly duty cycled by testing the
-	 * distributions of on and off times.
-	 * @param tempPeriods
-	 * @return
-	 */
-	private DutyCycleInfo assessDutyCycle(RecordingList tempPeriods) {
-		if (tempPeriods == null) {
-			return null;
-		}
-		int n = tempPeriods.size();
-		if (n < 2) {
-			return new DutyCycleInfo(false, 0,0,n);
-		}
-		double[] ons = new double[n-1]; // ignore the last one since it may be artificially shortened which is OK
-		double[] gaps = new double[n-1];
-		for (int i = 0; i < n-1; i++) {
-			ons[i] = tempPeriods.get(i).getDuration()/1000.;
-			gaps[i] = (tempPeriods.get(i+1).getRecordStart()-tempPeriods.get(i).getRecordStop())/1000.;
-		}
-		/* now look at how consistent those values are
-		 * But some data gets messed by small gaps, so want to 
-		 * remove outliers and concentrate on say 80% of the data. 
-		 */
-		ons = getDistributionCentre(ons, 80);
-		gaps = getDistributionCentre(gaps, 80);
-		Arrays.sort(gaps);
+//	/**
+//	 * Merge close recordings and discard ones that are too short. 
+//	 * @param tempPeriods all recording periods, may be from consecutive files. 
+//	 * @return merged list. 
+//	 */
+//	private RecordingList mergeRecordings(RecordingList tempPeriods) {
+//		// now go through those and merge into longer periods where there is no gap between files.
+//		if (tempPeriods == null) {
+//			return null;
+//		}
+//
+//		DeploymentExportOpts exportOptions = tethysControl.getDeploymentHandler().getDeploymentExportOptions();
+//
+//		ListIterator<RecordingPeriod> iterator = tempPeriods.listIterator();
+//		RecordingPeriod prevPeriod = null;
+//		while (iterator.hasNext()) {
+//			RecordingPeriod nextPeriod = iterator.next();
+//			long nextDur = nextPeriod.getRecordStop()-nextPeriod.getRecordStart();
+//			if (nextDur == 0) {
+//				continue;
+//			}
+//			if (prevPeriod != null) {
+//				long gap = nextPeriod.getRecordStart() - prevPeriod.getRecordStop();
+//				long prevDur = prevPeriod.getRecordStop()-prevPeriod.getRecordStart();
+//				if (gap < exportOptions.maxRecordingGapSeconds*1000) {
+//					// ignoring up to 3s gap or a sample error < 2%.Dunno if this is sensible or not.
+//					prevPeriod.setRecordStop(nextPeriod.getRecordStop());
+//					iterator.remove();
+//					nextPeriod = prevPeriod;
+//				}
+//			}
+//			prevPeriod = nextPeriod;
+//		}
+//		// now remove ones which are too short even after merging. 
+//		iterator = tempPeriods.listIterator();
+//		while (iterator.hasNext()) {
+//			RecordingPeriod nextPeriod = iterator.next();
+//			long duration = nextPeriod.getDuration();
+//			if (duration < exportOptions.minRecordingLengthSeconds*1000L) {
+//				iterator.remove();
+//			}
+//		}
+//
+//		return tempPeriods;
+//	}
 
 
-		STD std = new STD();
-		double onsMean = std.getMean(ons);
-		double onsSTD = std.getSTD(ons);
-		double gapsMean = std.getMean(gaps);
-		double gapsSTD = std.getSTD(gaps);
-		boolean dutyCycle = onsSTD/onsMean < .05 && gapsSTD/gapsMean < 0.05;
-		DutyCycleInfo cycleInfo = new DutyCycleInfo(dutyCycle, onsMean, gapsMean, tempPeriods.size());
-		return cycleInfo;
-	}
-
-	/**
-	 * Get the central part of a distribution without any outliers so 
-	 * that we can get a better assessment of duty cycle. 
-	 * @param data unsorted distribution data. 
-	 * @param percent percentage to include (half this removed from top and bottom)
-	 * @return
-	 */
-	private double[] getDistributionCentre(double[] data, double percent) {
-		if (data == null) {
-			return null;
-		}
-		Arrays.sort(data);
-		int nRem = (int) Math.round(data.length * (100-percent)/200);
-		int newLen = data.length-nRem*2;
-		double[] subdata = Arrays.copyOfRange(data, nRem, data.length-2*nRem);
-		if (subdata.length < 2) {
-			return data;
-		}
-		return subdata;
-	}
-
 
 	/**
 	 * Get data times from any other datamap, since this will generally match the acquisition anyway
@@ -360,16 +313,17 @@ public class EffortFunctions {
 			return null;
 		}
 		// get the times out of it. 
-		RecordingList recPeriods = new RecordingList();
+		RecordingList recPeriods = new RecordingList(bestMap.getDataMapName());
 		List<OfflineDataMapPoint> mapPoints = bestMap.getMapPoints();
 		for (OfflineDataMapPoint mapPoint : mapPoints) {
 			recPeriods.add(new RecordingPeriod(mapPoint.getStartTime(), mapPoint.getEndTime()));
+			recPeriods.add(mapPoint.getStartTime(), mapPoint.getEndTime());
 		}
 		return recPeriods;
 	}
 
 	private RecordingList extractTimesFromStatus(ArrayList<DaqStatusDataUnit> allStatusData) {
-		RecordingList tempPeriods = new RecordingList();
+		RecordingList tempPeriods = new RecordingList("Data acquisition status");
 		long dataStart = Long.MAX_VALUE;
 		long dataEnd = Long.MIN_VALUE;
 		Long lastStart = null;

src/tethys/deployment/RecordingList.java

@@ -1,13 +1,44 @@
 package tethys.deployment;
 
+import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.Iterator;
 
-public class RecordingList extends ArrayList<RecordingPeriod> {
+import PamUtils.PamCalendar;
+import pamMaths.STD;
+
+/**
+ * Information about periods of effort that might come from either the raw data recordings or 
+ * an analysis of binary data maps. 
+ * @author dg50
+ *
+ */
+public class RecordingList implements Serializable {
 
 	private static final long serialVersionUID = 1L;
 	
+	private ArrayList<RecordingPeriod> effortPeriods = new ArrayList();
+	
+	/**
+	 * Name / source of this list. 
+	 */
+	private String sourceName;
+
+	/**
+	 * @param sourceName
+	 */
+	public RecordingList(String sourceName) {
+		this.sourceName = sourceName;
+	}
+
+	public RecordingList(String sourceName, ArrayList<RecordingPeriod> selectedDeployments) {
+		this.sourceName = sourceName;
+		this.effortPeriods = selectedDeployments;
+	}
+
 	/**
 	 * Get the duration of the recording periods from start to end. 
 	 * @return
@@ -21,27 +52,27 @@ public class RecordingList extends ArrayList<RecordingPeriod> {
 	 * @return
 	 */
 	public long getStart() {
-		if (size() == 0) {
+		if (effortPeriods.size() == 0) {
 			return 0;
 		}
-		return get(0).getRecordStart();
+		return effortPeriods.get(0).getRecordStart();
 	}
 	
 	/**
 	 * get the end of the last in the list. 
 	 */
 	public long getEnd() {
-		if (size() == 0) {
+		if (effortPeriods.size() == 0) {
 			return 0;
 		}
-		return get(size()-1).getRecordStop();
+		return effortPeriods.get(effortPeriods.size()-1).getRecordStop();
 	}
 	
 	/**
 	 * Sort the list in ascending order. 
 	 */
 	public void sort() {
-		Collections.sort(this, new Comparator<RecordingPeriod>() {
+		Collections.sort(effortPeriods, new Comparator<RecordingPeriod>() {
 
 			@Override
 			public int compare(RecordingPeriod o1, RecordingPeriod o2) {
@@ -49,4 +80,172 @@ public class RecordingList extends ArrayList<RecordingPeriod> {
 			}
 		});
 	}
+	
+	/**
+	 * Get the coverage as a fraction. This is the sum of the individual periods divided 
+	 * by the start to end times
+	 * @return
+	 */
+	public double getCoverage() {
+		long cov = 0;
+		long durTot = 0;
+		if (effortPeriods.size() == 0) {
+			return 0;
+		}
+		Iterator<RecordingPeriod> it = effortPeriods.iterator();
+		while (it.hasNext()) {
+			RecordingPeriod rp = it.next();
+			cov += rp.getDuration();
+		}
+		durTot = getEnd()-getStart();
+		return (double) cov / (double) durTot;
+	}
+	
+	/**
+	 * Merge recording periods, with a max gap between periods in milliseconds. 
+	 * @param maxGap
+	 * @return the number of periods removed. 
+	 */
+	public int mergeRecordingPeriods(long maxGap) {
+		if (effortPeriods.size() < 2) {
+			return 0;
+		}
+		Iterator<RecordingPeriod> it = effortPeriods.iterator();
+		RecordingPeriod prev = it.next();
+		int removed = 0;
+		while (it.hasNext()) {
+			RecordingPeriod curr = it.next();
+			if (curr.getRecordStart() - prev.getRecordStop() <= maxGap) {
+				prev.setRecordStop(curr.getRecordStop());
+				it.remove();
+				removed++;
+			}
+			else {
+				prev = curr;
+			}
+		}
+		return removed;
+	}
+	
+	/**
+	 * Work out whether or not the data are evenly duty cycled by testing the
+	 * distributions of on and off times.
+	 * @param tempPeriods
+	 * @return
+	 */
+	public DutyCycleInfo assessDutyCycle() {
+		if (effortPeriods == null) {
+			return null;
+		}
+		int n = effortPeriods.size();
+		if (n < 2) {
+			return new DutyCycleInfo(false, 0,0,n);
+		}
+		double[] ons = new double[n-1]; // ignore the last one since it may be artificially shortened which is OK
+		double[] gaps = new double[n-1];
+		for (int i = 0; i < n-1; i++) {
+			ons[i] = effortPeriods.get(i).getDuration()/1000.;
+			gaps[i] = (effortPeriods.get(i+1).getRecordStart()-effortPeriods.get(i).getRecordStop())/1000.;
+		}
+		/* now look at how consistent those values are
+		 * But some data gets messed by small gaps, so want to 
+		 * remove outliers and concentrate on say 80% of the data. 
+		 */
+		ons = getDistributionCentre(ons, 80);
+		gaps = getDistributionCentre(gaps, 80);
+		Arrays.sort(gaps);
+
+		STD std = new STD();
+		double onsMean = std.getMean(ons);
+		double onsSTD = std.getSTD(ons);
+		double gapsMean = std.getMean(gaps);
+		double gapsSTD = std.getSTD(gaps);
+		boolean dutyCycle = onsSTD/onsMean < .05 && gapsSTD/gapsMean < 0.05;
+		DutyCycleInfo cycleInfo = new DutyCycleInfo(dutyCycle, onsMean, gapsMean, effortPeriods.size());
+		return cycleInfo;
+	}
+	/**
+	 * Get the central part of a distribution without any outliers so 
+	 * that we can get a better assessment of duty cycle. 
+	 * @param data unsorted distribution data. 
+	 * @param percent percentage to include (half this removed from top and bottom)
+	 * @return
+	 */
+	private double[] getDistributionCentre(double[] data, double percent) {
+		if (data == null) {
+			return null;
+		}
+		Arrays.sort(data);
+		int nRem = (int) Math.round(data.length * (100-percent)/200);
+		int newLen = data.length-nRem*2;
+		double[] subdata = Arrays.copyOfRange(data, nRem, data.length-2*nRem);
+		if (subdata.length < 2) {
+			return data;
+		}
+		return subdata;
+	}
+
+	/**
+	 * @return the sourceName
+	 */
+	public String getSourceName() {
+		return sourceName;
+	}
+
+	@Override
+	public String toString() {
+		if (effortPeriods.size() == 0) {
+			return "Empty recording list";
+		}
+		String str = String.format("%s: %s to %s, %3.1f%% coverage", getSourceName(),
+				PamCalendar.formatDBDateTime(getStart()), 
+				PamCalendar.formatDBDateTime(getEnd()), getCoverage()*100);
+		return str;
+	}
+	
+	/**
+	 * Get similarity to another recording list. 1 = identical, 0 means not even overlapping.
+	 * @param other other recording list. 
+	 * @return measure of similarity. 
+	 */
+	public double getSimilarity(RecordingList other) {
+		double sim1 = (double) other.duration() / (double) this.duration();
+		if (sim1 > 1) {
+			sim1 = 1./sim1;
+		}
+		long overlap = Math.min(other.getEnd(), this.getEnd()) - Math.max(other.getStart(), this.getStart());
+		overlap = Math.max(0,  overlap);
+		long longest = Math.max(other.duration(), this.duration());
+		double sim2 = (double) overlap / (double) longest;
+		
+		return Math.min(sim1, sim2);
+	}
+
+	/**
+	 * Add a recording period to the list. 
+	 * @param recordingPeriod
+	 */
+	public void add(RecordingPeriod recordingPeriod) {
+		effortPeriods.add(recordingPeriod);
+	}
+
+	/**
+	 * Add a recording period to the list. 
+	 * @param startTime
+	 * @param endTime
+	 */
+	public void add(long startTime, long endTime) {
+		add (new RecordingPeriod(startTime, endTime));
+	}
+
+	public int size() {
+		return effortPeriods.size();
+	}
+
+	/**
+	 * @return the effortPeriods
+	 */
+	public ArrayList<RecordingPeriod> getEffortPeriods() {
+		return effortPeriods;
+	}
 }

src/tethys/deployment/RecordingPeriod.java

@@ -1,5 +1,6 @@
 package tethys.deployment;
 
+import PamUtils.PamCalendar;
 import tethys.niluswraps.PDeployment;
 
 public class RecordingPeriod {
@@ -72,4 +73,10 @@ public class RecordingPeriod {
 		return selected;
 	}
 
+	@Override
+	public String toString() {
+		return String.format("%s to %s, %s", PamCalendar.formatDBDateTime(recordStart), 
+				PamCalendar.formatDBDateTime(recordStop), PamCalendar.formatDuration(getDuration()));
+	}
+	
 }

src/tethys/deployment/swing/EffortProblemDialog.java

@@ -0,0 +1,163 @@
+package tethys.deployment.swing;
+
+import java.awt.BorderLayout;
+import java.awt.GridBagConstraints;
+import java.awt.GridBagLayout;
+import java.awt.GridLayout;
+import java.awt.Window;
+
+import javax.swing.BoxLayout;
+import javax.swing.ButtonGroup;
+import javax.swing.JCheckBox;
+import javax.swing.JLabel;
+import javax.swing.JPanel;
+import javax.swing.JRadioButton;
+import javax.swing.border.TitledBorder;
+
+import PamUtils.PamCalendar;
+import PamView.dialog.PamDialog;
+import PamView.dialog.PamGridBagContraints;
+import tethys.deployment.DeploymentOverview;
+import tethys.deployment.RecordingList;
+
+/**
+ * Handle problems when binary and raw effort don't add up
+ * @author dg50
+ *
+ */
+public class EffortProblemDialog extends PamDialog {
+	
+	private JRadioButton useRaw, useBinary, useNeither;
+	
+	private JLabel generalInfo;
+	
+	private InfoSet[] infoSets = new InfoSet[2];
+
+	private RecordingList chosenList;
+
+	private DeploymentOverview deploymentOverview;
+	
+	private static EffortProblemDialog singleInstance;
+	
+	private static final String[] setNames = {"Raw data", "Binary data"};
+
+	private EffortProblemDialog(Window parentFrame) {
+		super(parentFrame, "Deployment Effort", false);
+		JPanel mainPanel = new JPanel();
+		mainPanel.setLayout(new BorderLayout());
+		mainPanel.setBorder(new TitledBorder("Effort information"));
+		String info = "<html>There is a mismatch between the time period covered by the raw<br>"
+				+ "data recordings and the time covered in the binary data.<br> "
+				+ "Select the one you wish to use, or Cancel and sort out your data<br>"
+				+ "prior to restarting the Tethys export process</html>";
+		generalInfo = new JLabel(info);
+//		generalInfo.setBorder(new TitledBorder("General"));
+		mainPanel.add(generalInfo, BorderLayout.NORTH);
+		JPanel botPanel = new JPanel(new GridLayout(2, 1));
+		mainPanel.add(botPanel, BorderLayout.CENTER);
+		ButtonGroup bg = new ButtonGroup();
+		for (int i = 0; i < 2; i++) {
+			GridBagConstraints c = new PamGridBagContraints();
+			JPanel subPanel = new JPanel(new GridBagLayout());
+			botPanel.add(subPanel);
+			infoSets[i] = new InfoSet(setNames[i]);
+			c.gridwidth = 2;
+			subPanel.add(infoSets[i].name, c);
+			c.gridx += c.gridwidth;
+			subPanel.add(infoSets[i].select, c);
+			c.gridx = 0;
+			c.gridy++;
+			c.gridwidth = 1;
+			subPanel.add(new JLabel("Start: ", JLabel.RIGHT), c);
+			c.gridx++;
+			subPanel.add(infoSets[i].start, c);
+			c.gridx++;
+			subPanel.add(new JLabel("End: ", JLabel.RIGHT), c);
+			c.gridx++;
+			subPanel.add(infoSets[i].end, c);
+			c.gridy++;
+			c.gridx = 0;
+			subPanel.add(new JLabel("Duration: ", JLabel.RIGHT), c);
+			c.gridx++;
+			subPanel.add(infoSets[i].duration, c);
+			c.gridx++;
+			subPanel.add(new JLabel("Coverage: ", JLabel.RIGHT), c);
+			c.gridx++;
+			subPanel.add(infoSets[i].occupancy, c);
+
+			bg.add(infoSets[i].select);
+		}
+		
+		setDialogComponent(mainPanel);
+		setResizable(true);
+	}
+	
+	public static RecordingList showDialog(Window parentFrame, DeploymentOverview deploymentOverview) {
+		singleInstance = new EffortProblemDialog(parentFrame);
+		singleInstance.setData(deploymentOverview);
+		singleInstance.setVisible(true);
+		return singleInstance.chosenList;
+	}
+
+	private void setData(DeploymentOverview deploymentOverview) {
+		this.deploymentOverview = deploymentOverview;
+		RecordingList rl;
+		for (int i = 0; i < 2; i++) {
+			if (i == 0) {
+				rl = deploymentOverview.getRawDataList();
+			}
+			else {
+				rl = deploymentOverview.getBinaryDataList();
+			}
+			infoSets[i].start.setText(PamCalendar.formatDBDateTime(rl.getStart()));
+			infoSets[i].end.setText(PamCalendar.formatDBDateTime(rl.getEnd()));
+			infoSets[i].duration.setText(PamCalendar.formatDuration(rl.duration()));
+			infoSets[i].occupancy.setText(String.format("%3.0f%%", rl.getCoverage()*100.));
+		}
+		invalidate();
+		pack();
+	}
+
+	@Override
+	public boolean getParams() {
+		if (infoSets[0].select.isSelected()) {
+			chosenList = deploymentOverview.getRawDataList();
+			return true;
+		}
+		if (infoSets[1].select.isSelected()) {
+			chosenList = deploymentOverview.getBinaryDataList();
+			return true;
+		}
+		return false;
+	}
+
+	@Override
+	public void cancelButtonPressed() {
+		// TODO Auto-generated method stub
+
+	}
+
+	@Override
+	public void restoreDefaultSettings() {
+		// TODO Auto-generated method stub
+
+	}
+	
+	private class InfoSet {
+		JLabel name, start, end, duration, occupancy;
+		JCheckBox select;
+		/**
+		 * 
+		 */
+		public InfoSet(String name) {
+			super();
+			this.name = new JLabel(name);
+			this.start = new JLabel("                   ");
+			this.end = new JLabel("                        ");
+			this.select = new JCheckBox("Select " + name);
+			duration = new JLabel(" ");
+			occupancy = new JLabel(" ");
+		}
+	}
+
+}

src/tethys/deployment/swing/RecordingGapDialog.java

@@ -36,7 +36,7 @@ public class RecordingGapDialog extends PamDialog {
 		c.gridx++;
 		mainPanel.add(new JLabel(" seconds", JLabel.RIGHT), c);
 		
-		maxGap.setToolTipText("Maximum gap between recording periods. Periods with a gap less than this will be counted as one");
+		maxGap.setToolTipText("Maximum gap between recording periods. Sequential periods with a gap less than this will be counted as one");
 		minLength.setToolTipText("Minimum recording length. Recording sections shorter than this will be ignored");
 		
 		setDialogComponent(mainPanel);
@@ -80,6 +80,7 @@ public class RecordingGapDialog extends PamDialog {
 	@Override
 	public void restoreDefaultSettings() {
 		DeploymentExportOpts defaults = new DeploymentExportOpts();
+		setParams(defaults);
 	}
 
 }

src/tethys/detection/DetectionExportProgress.java

@@ -17,6 +17,7 @@ public class DetectionExportProgress {
 	public int exportCount;
 	public int skipCount;
 	public int state;
+	public int totalDeployments, deploymentsDone;
 	
 	public DetectionExportProgress(PDeployment currentDeployment, Detections currentDetections, long lastUnitTime,
 			long totalUnits, int exportCount, int skipCount, int state) {

src/tethys/detection/DetectionsHandler.java

@@ -11,7 +11,6 @@ import PamController.PamguardVersionInfo;
 import PamModel.PamPluginInterface;
 import PamUtils.PamCalendar;
 import PamView.dialog.PamDialog;
-import PamView.dialog.warn.WarnOnce;
 import PamguardMVC.PamDataBlock;
 import PamguardMVC.PamDataUnit;
 import PamguardMVC.PamProcess;
@@ -32,6 +31,7 @@ import nilus.Detections;
 import nilus.GranularityEnumType;
 import nilus.Helper;
 import tethys.Collection;
+import tethys.CollectionHandler;
 import tethys.TethysControl;
 import tethys.TethysTimeFuncs;
 import tethys.dbxml.DBXMLConnect;
@@ -39,7 +39,6 @@ import tethys.dbxml.TethysException;
 import tethys.deployment.DeploymentHandler;
 import tethys.niluswraps.PDeployment;
 import tethys.niluswraps.PDetections;
-import tethys.output.DatablockSynchInfo;
 import tethys.output.StreamExportParams;
 import tethys.output.TethysExportParams;
 import tethys.pamdata.TethysDataProvider;
@@ -54,9 +53,7 @@ import tethys.swing.export.DetectionsExportWizard;
  * @author dg50
  *
  */
-public class DetectionsHandler {
-
-	private TethysControl tethysControl;
+public class DetectionsHandler extends CollectionHandler {
 
 	public int uniqueDetectionsId=1;
 	public int uniqueDetectionId;
@@ -65,12 +62,14 @@ public class DetectionsHandler {
 
 	private ExportWorker exportWorker;
 
+	public static final String helpPoint = "utilities.tethys.docs.detect_localize";
+
 	/**
 	 * 
 	 * @param tethysControl
 	 */
 	public DetectionsHandler(TethysControl tethysControl) {
-		super();
+		super(tethysControl, Collection.Detections);
 		this.tethysControl = tethysControl;
 	}
 
@@ -296,6 +295,9 @@ public class DetectionsHandler {
 			viewerLoadPolicy = ViewerLoadPolicy.LOAD_UTCNORMAL;
 		}
 		GranularityHandler granularityHandler = GranularityHandler.getHandler(streamExportParams.granularity, tethysControl, dataBlock, exportParams, streamExportParams);
+		int totalMaps = 0;
+		int totalMappedPoints = 0;
+		int totalLoadedDatas = 0;
 		for (PDeployment deployment : deployments) {
 			int documentCount = 0;
 			prog = new DetectionExportProgress(deployment, null,
@@ -311,6 +313,7 @@ public class DetectionsHandler {
 					prog = new DetectionExportProgress(deployment, null,
 							lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_CANCELED);
 					exportObserver.update(prog);
+					break;
 				}
 
 				if (mapPoint.getEndTime() < deployment.getAudioStart()) {
@@ -319,10 +322,13 @@ public class DetectionsHandler {
 				if (mapPoint.getStartTime() >= deployment.getAudioEnd()) {
 					break;
 				}
+				totalMaps ++;
+				totalMappedPoints += mapPoint.getNDatas();
 				dataBlock.loadViewerData(mapPoint.getStartTime(), mapPoint.getEndTime(), null);
 				ArrayList<PamDataUnit> dataCopy = dataBlock.getDataCopy(deployment.getAudioStart(), deployment.getAudioEnd(), true, dataSelector);
-//				System.out.printf("%d loaded from %s to %s %d kept\n", dataBlock.getUnitsCount(), PamCalendar.formatDateTime(mapPoint.getStartTime()),
-//						PamCalendar.formatDateTime(mapPoint.getEndTime()), dataCopy.size());
+				totalLoadedDatas += dataCopy.size();
+				System.out.printf("%d loaded from %s to %s %d kept\n", dataBlock.getUnitsCount(), PamCalendar.formatDateTime(mapPoint.getStartTime()),
+						PamCalendar.formatDateTime(mapPoint.getEndTime()), dataCopy.size());
 				skipCount += dataBlock.getUnitsCount() - dataCopy.size();
 				for (PamDataUnit dataUnit : dataCopy) {
 					/*
@@ -353,7 +359,9 @@ public class DetectionsHandler {
 				if (viewerLoadPolicy == ViewerLoadPolicy.LOAD_ALWAYS_EVERYTHING) {
 					break;
 				}
-
+				if (!activeExport) {
+					return 0;
+				}
 			}
 			Detection dets[] = granularityHandler.cleanup(deployment.getAudioEnd());
 			if (dets != null) {
@@ -405,10 +413,6 @@ public class DetectionsHandler {
 			exportObserver.update(prog);
 			granularityHandler.prepare(deployment.getAudioStart());
 
-			if (currentDetections == null) {
-				currentDetections = startDetectionsDocument(deployment, dataBlock, streamExportParams);
-				currentDetections.getEffort().setStart(TethysTimeFuncs.xmlGregCalFromMillis(deployment.getAudioStart()));
-			}
 			// export everything in that deployment.
 			// need to loop through all map points in this interval.
 			List<OfflineDataMapPoint> mapPoints = dataMap.getMapPoints();
@@ -417,6 +421,13 @@ public class DetectionsHandler {
 					prog = new DetectionExportProgress(deployment, currentDetections,
 							lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_CANCELED);
 					exportObserver.update(prog);
+					break;
+				}
+
+				if (currentDetections == null) {
+					// needed in inner loop in case doc gets written at 500000.
+					currentDetections = startDetectionsDocument(deployment, dataBlock, streamExportParams);
+					currentDetections.getEffort().setStart(TethysTimeFuncs.xmlGregCalFromMillis(deployment.getAudioStart()));
 				}
 				
 				if (mapPoint.getEndTime() < deployment.getAudioStart()) {
@@ -453,7 +464,7 @@ public class DetectionsHandler {
 						lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_GATHERING);
 				exportObserver.update(prog);
 
-				if (documentCount > 500000 && mapPoint != dataMap.getLastMapPoint()) {
+				if (documentCount > 50000000 && mapPoint != dataMap.getLastMapPoint()) {
 					prog = new DetectionExportProgress(deployment, currentDetections,
 							lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_WRITING);
 					exportObserver.update(prog);
@@ -469,8 +480,14 @@ public class DetectionsHandler {
 				if (viewerLoadPolicy == ViewerLoadPolicy.LOAD_ALWAYS_EVERYTHING) {
 					break;
 				}
+				if (!activeExport) {
+					break;
+				}
 			}
 
+			if (!activeExport) {
+				return DetectionExportProgress.STATE_CANCELED;
+			}
 
 			if (currentDetections != null) {
 				Detection dets[] = granularityHandler.cleanup(deployment.getAudioEnd());
@@ -552,7 +569,7 @@ public class DetectionsHandler {
 		supportSoft.setVersion(getSupportSoftwareVersion(dataBlock));
 		supSoft.add(supportSoft);
 		detections.setAlgorithm(algorithm);
-		detections.setUserId("Unknown user");
+		detections.setUserId("PAMGuard user");
 		detections.setEffort(getDetectorEffort(deployment, dataBlock, exportParams));
 
 		return detections;
@@ -599,13 +616,16 @@ public class DetectionsHandler {
 		protected Integer doInBackground() throws Exception {
 			Integer ans = null;
 			try {
-				int count = countDetections(dataBlock, exportParams, exportObserver);
-				String msg = String.format("Do you want to go ahead and output %d %s detections to Tethys?", 
-						count, exportParams.granularity);
-				int doit = WarnOnce.showWarning("Tethys Detections Export", msg, WarnOnce.OK_CANCEL_OPTION);
-				if (doit == WarnOnce.OK_OPTION) {
+//				int count = countDetections(dataBlock, exportParams, exportObserver);
+//				if (activeExport == false) {
+//					return 0;
+//				}
+//				String msg = String.format("Do you want to go ahead and output %d %s detections to Tethys?", 
+//						count, exportParams.granularity);
+//				int doit = WarnOnce.showWarning("Tethys Detections Export", msg, WarnOnce.OK_CANCEL_OPTION);
+//				if (doit == WarnOnce.OK_OPTION) {
 					ans = exportDetections(dataBlock, exportParams, this);
-				}
+//				}
 			}
 			catch (Exception e) {
 				e.printStackTrace();
@@ -661,4 +681,10 @@ public class DetectionsHandler {
 		DetectionsExportWizard.showDialog(tethysControl.getGuiFrame(), tethysControl, dataBlock);
 
 	}
+
+
+	@Override
+	public String getHelpPoint() {
+		return helpPoint;
+	}
 }

src/tethys/localization/LocalizationHandler.java

@@ -0,0 +1,17 @@
+package tethys.localization;
+
+import nilus.CylindricalCoordinateType;
+import nilus.LocalizationType;
+import nilus.Localize.Effort.CoordinateReferenceSystem;
+
+public class LocalizationHandler {
+
+	
+	public LocalizationType getLoc() {
+		LocalizationType lt = new LocalizationType();
+		CylindricalCoordinateType cct = new CylindricalCoordinateType();
+//		cct.set
+		CoordinateReferenceSystem cr;
+		return null;
+	}
+}

src/tethys/niluswraps/NilusSettingsWrapper.java

@@ -129,7 +129,9 @@ public class NilusSettingsWrapper<T extends Object> implements Serializable, Clo
 			Document doc = builder.parse(new InputSource(new StringReader(xmlString)));
 			return doc;
 		} catch (Exception e) {
-			e.printStackTrace();
+			System.out.println(e.getMessage());
+			System.out.println("Nilus Settings wrapper - Error parsing string\n" + xmlString);
+//			e.printStackTrace();
 		}
 		return null;
 	}

src/tethys/output/StreamExportParams.java

@@ -98,7 +98,11 @@ public class StreamExportParams implements Serializable {
 	 * @return
 	 */
 	public DescriptionType getNilusDetectionDescription() {
-		return getDetectionDescription().getDescription();
+		WrappedDescriptionType desc = getDetectionDescription();
+		if (desc == null) {
+			return null;
+		}
+		return desc.getDescription();
 	}
 
 }

src/tethys/output/TethysExportParams.java

@@ -46,6 +46,8 @@ public class TethysExportParams implements Serializable, Cloneable{
 
 	public boolean listDocsInPamguard;
 
+	private String effortSourceName;
+	
 
 	/**
 	 * @return the datasetName
@@ -121,6 +123,22 @@ public class TethysExportParams implements Serializable, Cloneable{
 		return streamParamsMap.get(longDataName);
 	}
 
+	/**
+	 * Source name for type of effort. 
+	 * @param sourceName
+	 */
+	public void setEffortSourceName(String sourceName) {
+		this.effortSourceName = sourceName;
+	}
+
+	/**
+	 * Source name for type of effort. 
+	 * @return the effortSourceName
+	 */
+	public String getEffortSourceName() {
+		return effortSourceName;
+	}
+
 
 
 }

src/tethys/species/DataBlockSpeciesManager.java

@@ -97,7 +97,10 @@ abstract public class DataBlockSpeciesManager<T extends PamDataUnit> {
 	public SpeciesMapItem getSpeciesItem(T dataUnit) {
 		String speciesString = getSpeciesCode(dataUnit);
 		if (speciesString == null) {
-			return getDefaultDefaultSpecies();
+			SpeciesMapItem def = getDefaultDefaultSpecies();
+			if (def != null) {
+				speciesString = def.getPamguardName();
+			}
 		}
 		DataBlockSpeciesMap speciesMap = getDatablockSpeciesMap();
 		if (speciesMap == null) {
@@ -127,7 +130,26 @@ abstract public class DataBlockSpeciesManager<T extends PamDataUnit> {
 		if (allCodes.size() == 0) {
 			allCodes.add("Unknown");
 		}
-		return allCodes;
+		return makeUniqueList(allCodes);
+	}
+	
+	/**
+	 * Make sure all entries in an array list are unique. 
+	 * @param list
+	 * @return updated list. 
+	 */
+	public ArrayList<String> makeUniqueList(ArrayList<String> list) {
+		if (list == null) {
+			return null;
+		}
+		ArrayList<String> newList = new ArrayList();
+		for (String aStr : list) {
+			if (newList.contains(aStr)) {
+				continue;
+			}
+			newList.add(aStr);
+		}
+		return newList;
 	}
 	
 	public DataBlockSpeciesMap getDatablockSpeciesMap() {

src/tethys/species/ITISFunctions.java

@@ -70,7 +70,7 @@ public class ITISFunctions {
 //		PAMGuardXMLPreview xmlPreview = new PAMGuardXMLPreview(null, "returned", qResult.queryResult)
 		PamguardXMLWriter pamXMLWriter = PamguardXMLWriter.getXMLWriter();
 		String fDoc = pamXMLWriter.getAsString(doc, true);
-		System.out.println(fDoc);
+//		System.out.println(fDoc);
 		
 		String tsn = dbQueries.getElementData(docEl, "tsn");
 		if (tsn == null) {
@@ -91,7 +91,46 @@ public class ITISFunctions {
 		return new TethysITISResult(itisCode, taxunit, latin, vernacular);
 	}
 	
+	/**
+	 * Search species codes. If the search term is a valid Integer number
+	 * then it's assumed to be an ITIS code and the function should
+	 * return a single map item. If it's non-integer, it's assumed to 
+	 * be a common or latin name search
+	 * @param searchTerm
+	 * @return array list of possible matches. 
+	 */
 	public ArrayList<SpeciesMapItem> searchSpecies(String searchTerm) {
+		Integer intVal = null;
+		try {
+			intVal = Integer.valueOf(searchTerm);
+		}
+		catch (NumberFormatException e) {
+			intVal = null;
+		}
+		if (intVal != null) {
+			return searchCodes(intVal);
+		}
+		else { // assume name search
+			return searchNames(searchTerm);
+		}
+	}
+	
+	private ArrayList<SpeciesMapItem> searchCodes(Integer intCode) {
+		ArrayList<SpeciesMapItem> mapItems = new ArrayList();
+		TethysITISResult result = getITISInformation(intCode);
+		if (result != null) {
+			mapItems.add(new SpeciesMapItem(intCode, "", "", result.getLatin(), result.getVernacular()));
+		}
+		return mapItems;
+	}
+
+	/**
+	 * Search common and latin names for partial matched of the search term
+	 * and return an array list of all possible matches. 
+	 * @param searchTerm
+	 * @return
+	 */
+	public ArrayList<SpeciesMapItem> searchNames(String searchTerm) {
 		ArrayList<SpeciesMapItem> items = new ArrayList<SpeciesMapItem>();		
 		String xQ = "let $target := \"thespeciessearchterm\" \r\n"
 				+ "return\r\n"

src/tethys/species/ITISTypes.java

@@ -7,8 +7,8 @@ package tethys.species;
  */
 public class ITISTypes {
 
-	public static final int OTHER = 0;
-	public static final int ANTHROPOGENIC = 1;
+	public static final int OTHER = -10;
+	public static final int ANTHROPOGENIC = 1758;
 	
 	public static final String getName(int code) {
 		switch (code) {

src/tethys/species/SpeciesMapItem.java

@@ -41,6 +41,14 @@ public class SpeciesMapItem implements Serializable, Cloneable {
 	 */
 	private String callType;
 
+	/**
+	 * 
+	 * @param itisCode
+	 * @param callType
+	 * @param pamguardName
+	 * @param latinName
+	 * @param commonName
+	 */
 	public SpeciesMapItem(int itisCode, String callType, String pamguardName, String latinName, String commonName) {
 		super();
 		this.itisCode = itisCode;
@@ -50,6 +58,12 @@ public class SpeciesMapItem implements Serializable, Cloneable {
 		this.commonName = commonName;
 	}
 
+	/**
+	 * 
+	 * @param itisCode
+	 * @param callType
+	 * @param pamguardName
+	 */
 	public SpeciesMapItem(int itisCode, String callType, String pamguardName) {
 		super();
 		this.itisCode = itisCode;

src/tethys/species/swing/DataBlockSpeciesDialog.java

@@ -50,15 +50,6 @@ public class DataBlockSpeciesDialog extends PamDialog {
 				+ "<br>When known, a call or sound type should "
 				+ "be specified (see help for more information).</html>";
 		nPanel.add(BorderLayout.CENTER, new JLabel(otherMsg , JLabel.LEFT));		
-//		JPanel nwBit = new JPanel(new FlowLayout());
-//		JButton exportButton = new JButton("Export");
-//		exportButton.addActionListener(SpeciesMapManager.getInstance().getExportAction(parentFrame));
-//		nwBit.add(exportButton);
-//		JButton importButton = new JButton("Import");
-//		importButton.addActionListener(SpeciesMapManager.getInstance().getImportAction(parentFrame));
-//		nwBit.add(importButton);
-//		nPanel.add(BorderLayout.WEST, nwBit);
-		
 		
 		mainPanel.add(BorderLayout.NORTH, nPanel);
 		setDialogComponent(mainPanel);

src/tethys/species/swing/SpeciesSearchDialog.java

@@ -104,11 +104,11 @@ public class SpeciesSearchDialog extends PamDialog {
 		setResizable(true);
 		setDialogComponent(mainPanel);
 	}
-	public static SpeciesMapItem showDialog(Window parentFrame, TethysControl tethysControl) {
+	public static SpeciesMapItem showDialog(Window parentFrame, TethysControl tethysControl, Integer currentCode) {
 		if (singleInstance == null) {
 			singleInstance = new SpeciesSearchDialog(parentFrame, tethysControl);
 		}
-		singleInstance.setParams();
+		singleInstance.setParams(currentCode);
 		singleInstance.setVisible(true);
 		return singleInstance.selectedItem;
 	}
@@ -131,6 +131,9 @@ public class SpeciesSearchDialog extends PamDialog {
 	
 	public void setMapItems(ArrayList<SpeciesMapItem> newMapItems) {
 		this.speciesMapItems = newMapItems;
+		if (newMapItems != null && newMapItems.size() == 1) {
+			setSelectedItem(newMapItems.get(0));
+		}
 		tableModel.fireTableDataChanged();
 	}
 
@@ -188,14 +191,20 @@ public class SpeciesSearchDialog extends PamDialog {
 		
 	}
 	
-	private void setParams() {
-		searchText.setText(null);
-		clearResults();
+	private void setParams(Integer currentCode) {
+		if (currentCode == null) {
+			searchText.setText(null);
+			clearResults();
+		}
+		else {
+			searchText.setText(currentCode.toString());
+			searchTethys();
+		}
 	}
 
 	private void clearResults() {
 		speciesMapItems = null;
-		selectedItem = null;
+		setSelectedItem(null);
 	}
 	@Override
 	public boolean getParams() {
@@ -216,6 +225,10 @@ public class SpeciesSearchDialog extends PamDialog {
 
 	}
 	
+	private void enableControls() {
+		getOkButton().setEnabled(selectedItem != null);
+	}
+	
 	private class TableMouse extends MouseAdapter {
 
 		@Override
@@ -225,12 +238,18 @@ public class SpeciesSearchDialog extends PamDialog {
 			}
 			int selectedRow = resultTable.getSelectedRow();
 			if (selectedRow >= 0 && selectedRow < speciesMapItems.size()) {
-				selectedItem = speciesMapItems.get(selectedRow);
+				setSelectedItem(speciesMapItems.get(selectedRow));
 			}
 			tableModel.fireTableDataChanged();
 		}
 		
 	}
+	
+	private void setSelectedItem(SpeciesMapItem selItem) {
+		this.selectedItem = selItem;
+		enableControls();
+	}
+	
 	private class DataModel extends AbstractTableModel {
 		
 		private String[] colNames = {"Select", "TSN", "Name", "Common Name"};

src/tethys/species/swing/SpeciesSubPanel.java

@@ -114,12 +114,12 @@ public class SpeciesSubPanel {
 		}
 		ITISFunctions itisFunctions = tethysControl.getItisFunctions();
 		String itisString = this.itisCode.getText();
-		if (itisString == null || itisString.length() == 0) {
+//		if (itisString == null || itisString.length() == 0) {
 			searchForCode(tethysControl, itisFunctions);
-		}
-		else {
-			getCodeInformation(tethysControl, itisFunctions, itisString);
-		}
+//		}
+//		else {
+//			getCodeInformation(tethysControl, itisFunctions, itisString);
+//		}
 //		System.out.println(itisInfo);
 	}
 
@@ -144,7 +144,15 @@ public class SpeciesSubPanel {
 	}
 
 	private void searchForCode(TethysControl tethysControl, ITISFunctions itisFunctions) {
-		SpeciesMapItem speciesItem = SpeciesSearchDialog.showDialog(tethysControl.getGuiFrame(), tethysControl);
+		Integer currentCode = null;
+		try {
+			currentCode = Integer.valueOf(itisCode.getText());
+		}
+		catch (NumberFormatException e) {
+			
+		}
+		
+		SpeciesMapItem speciesItem = SpeciesSearchDialog.showDialog(tethysControl.getGuiFrame(), tethysControl, currentCode);
 		if (speciesItem != null) {
 			itisCode.setText(String.format("%d", speciesItem.getItisCode()));
 			latinName.setText(speciesItem.getLatinName());

src/tethys/swing/DatablockDetectionsPanel.java

@@ -19,6 +19,8 @@ import javax.swing.table.AbstractTableModel;
 import javax.swing.table.JTableHeader;
 import javax.xml.datatype.XMLGregorianCalendar;
 
+import PamUtils.worker.PamWorkWrapper;
+import PamUtils.worker.PamWorker;
 import PamView.PamGui;
 import PamView.dialog.warn.WarnOnce;
 import PamView.tables.SwingTableColumnWidths;
@@ -42,7 +44,7 @@ import tethys.niluswraps.PDetections;
  * @author dg50
  *
  */
-public class DatablockDetectionsPanel extends TethysGUIPanel implements StreamTableObserver {
+public class DatablockDetectionsPanel extends TethysGUIPanel implements StreamTableObserver, PamWorkWrapper<String> {
 
 	private JPanel mainPanel;
 
@@ -118,15 +120,37 @@ public class DatablockDetectionsPanel extends TethysGUIPanel implements StreamTa
 
 	@Override
 	public void selectDataBlock(PamDataBlock dataBlock) {
+		if (this.dataBlock == dataBlock) {
+			return; // stops lots of requerying, which matters when database is large. 
+		}
 		this.dataBlock = dataBlock;
-		dataBlockName.setText(dataBlock.getLongDataName());
-		streamDetectionsSummary = getTethysControl().getDetectionsHandler().getStreamDetections(dataBlock);
+		if (dataBlock == null) {
+			dataBlockName.setText("Select data in panel on the left");
+		}
+		else {
+			dataBlockName.setText(dataBlock.getLongDataName());
+		}
+		// need to re-thread this to stop user panicing that nothing is happening. 
+		PamWorker w = new PamWorker<String>(this, getTethysControl().getGuiFrame(), 0, "Searching database");
+		w.start();
+	}
+
+	@Override
+	public void taskFinished(String result) {
 		tableModel.fireTableDataChanged();
 	}
 
+	@Override
+	public String runBackgroundTask(PamWorker<String> pamWorker) {
+		streamDetectionsSummary = getTethysControl().getDetectionsHandler().getStreamDetections(dataBlock);
+		return null;
+	}
+
 	@Override
 	public void updateState(TethysState tethysState) {
 		if (dataBlock != null) {
+			PamDataBlock currBlock = dataBlock;
+			selectDataBlock(null);
 			selectDataBlock(dataBlock);
 		}
 	}

src/tethys/swing/DatablockSynchPanel.java

@@ -1,6 +1,7 @@
 package tethys.swing;
 
 import java.awt.BorderLayout;
+import java.awt.Color;
 import java.awt.event.ActionEvent;
 import java.awt.event.ActionListener;
 import java.awt.event.ComponentEvent;
@@ -13,6 +14,7 @@ import java.util.ArrayList;
 
 import javax.swing.JButton;
 import javax.swing.JComponent;
+import javax.swing.JLabel;
 import javax.swing.JMenuItem;
 import javax.swing.JPanel;
 import javax.swing.JPopupMenu;
@@ -36,9 +38,9 @@ import tethys.niluswraps.PDeployment;
 import tethys.output.DatablockSynchInfo;
 import tethys.species.DataBlockSpeciesManager;
 
-public class DatablockSynchPanel extends TethysGUIPanel {
+public class DatablockSynchPanel extends TethysExportPanel {
 	
-	public JPanel mainPanel;
+//	public JPanel mainPanel;
 	
 	private JTable synchTable;
 	
@@ -48,39 +50,46 @@ public class DatablockSynchPanel extends TethysGUIPanel {
 	
 	private ArrayList<StreamTableObserver> tableObservers = new ArrayList<>();
 	
-	private JButton exportButton;
+//	private TippedButton exportButton;
+
+//	private JLabel exportWarning;
+	
 	
 	public DatablockSynchPanel(TethysControl tethysControl) {
-		super(tethysControl);
-		mainPanel = new PamPanel(new BorderLayout());
+		super(tethysControl, tethysControl.getDetectionsHandler(), false);
+//		mainPanel = new PamPanel(new BorderLayout());
+		JPanel mainPanel = getMainPanel();
 		mainPanel.setBorder(new TitledBorder("PAMGuard data blocks"));
 		synchTableModel = new SynchTableModel();
 		synchTable = new JTable(synchTableModel);
 		new SwingTableColumnWidths(tethysControl.getUnitName()+"SynchTable", synchTable);
 		JScrollPane scrollPane = new JScrollPane(synchTable);
 		mainPanel.add(BorderLayout.CENTER, scrollPane);
-		PamPanel ctrlPanel = new PamPanel(new BorderLayout());
-		exportButton = new JButton("Export ...");
-		ctrlPanel.add(BorderLayout.WEST, exportButton);
-		mainPanel.add(BorderLayout.NORTH, ctrlPanel);
+//		PamPanel ctrlPanel = new PamPanel(new BorderLayout());
+//		exportButton = new TippedButton("Export ...", "Export Detections document");
+//		exportWarning = new JLabel("  ");
+//		exportWarning.setForeground(Color.RED);
+//		ctrlPanel.add(BorderLayout.WEST, exportButton);
+//		ctrlPanel.add(BorderLayout.CENTER, exportWarning);
+//		mainPanel.add(BorderLayout.NORTH, ctrlPanel);
 
 		
 		synchTable.addMouseListener(new MouseActions());
 		synchTable.addKeyListener(new KeyActions());
 		
-		exportButton.addActionListener(new ActionListener() {
-			@Override
-			public void actionPerformed(ActionEvent e) {
-				exportData();
-			}
-		});
+//		exportButton.addActionListener(new ActionListener() {
+//			@Override
+//			public void actionPerformed(ActionEvent e) {
+//				exportData();
+//			}
+//		});
 		enableExportButton();
 	}
 
-	@Override
-	public JComponent getComponent() {
-		return mainPanel;
-	}
+//	@Override
+//	public JComponent getComponent() {
+//		return mainPanel;
+//	}
 	
 	private class KeyActions extends KeyAdapter {
 		@Override
@@ -141,13 +150,50 @@ public class DatablockSynchPanel extends TethysGUIPanel {
 	}
 
 	private void enableExportButton() {
+		if (!getTethysControl().isServerOk()) {
+			disableExport("Tethys Server not running");
+			return;
+		}
 		int[] rows = synchTable.getSelectedRows();
-		boolean en = rows != null && rows.length == 1;
 		ArrayList<PDeployment> deployments = getTethysControl().getDeploymentHandler().getMatchedDeployments();
 		if (deployments == null || deployments.size() == 0) {
-			en = false;
+			disableExport("No Deployment document(s). Export Deployments prior to exporting Detections");
+			return;
 		}
-		exportButton.setEnabled(getTethysControl().isServerOk() & en);
+		boolean en = rows != null && rows.length == 1;
+		if (!en) {
+			disableExport("No PAMGuard datablock selected (click a row on the table below)");
+			return;
+		}
+
+		PamDataBlock dataBlock = dataBlockSynchInfo.get(rows[0]).getDataBlock();
+		String mapError = checkSpeciesManager(dataBlock);
+		if (mapError != null) {
+			disableExport("Unable to export due to species map error: " + mapError + ". Right click table row to edit species list");
+			return;
+		}
+		
+		enableExport(true);
+	}
+	
+//	public void disableExport(String reason) {
+//		if (reason == null) {
+//			exportButton.setEnabled(true);
+//			exportWarning.setText(null);
+//		}
+//		else {
+//			exportButton.disable(reason);
+//			exportWarning.setText("   " + reason);
+//		}
+//	}
+	
+	private String checkSpeciesManager(PamDataBlock dataBlock) {
+		DataBlockSpeciesManager spManager = dataBlock.getDatablockSpeciesManager();
+		if (spManager == null) {
+			return "No species manager";
+		}
+		String error = spManager.checkSpeciesMapError();
+		return error;
 	}
 
 	public void showPopup(MouseEvent e, int row) {
@@ -259,4 +305,15 @@ public class DatablockSynchPanel extends TethysGUIPanel {
 		}
 		
 	}
+
+	@Override
+	protected void exportButtonPressed(ActionEvent e) {
+		exportData();
+	}
+
+	@Override
+	protected void optionsButtonPressed(ActionEvent e) {
+		// TODO Auto-generated method stub
+		
+	}
 }

src/tethys/swing/DeploymentExportPanel.java

@@ -34,6 +34,7 @@ import tethys.TethysState;
 import tethys.TethysState.StateType;
 import tethys.dbxml.DBXMLConnect;
 import tethys.deployment.DeploymentHandler;
+import tethys.deployment.RecordingList;
 import tethys.deployment.RecordingPeriod;
 import tethys.niluswraps.PDeployment;
 
@@ -241,7 +242,9 @@ public class DeploymentExportPanel extends TethysGUIPanel implements DeploymentT
 		if (selectedDeployments == null || selectedDeployments.size() == 0) {
 			return;
 		};
-		getTethysControl().getDeploymentHandler().exportDeployments(selectedDeployments);
+		// need to turn that list back into a RecordingList object. 
+		RecordingList tempList = new RecordingList("eport list", selectedDeployments);
+		getTethysControl().getDeploymentHandler().exportDeployments(tempList);
 	}
 	
 

src/tethys/swing/DeploymentsPanel.java

@@ -16,81 +16,38 @@ import javax.swing.border.TitledBorder;
 import PamView.panel.PamPanel;
 import tethys.TethysControl;
 import tethys.TethysState;
+import tethys.calibration.CalibrationHandler;
 import tethys.deployment.DeploymentHandler;
+import tethys.deployment.RecordingList;
 import tethys.deployment.RecordingPeriod;
 
-public class DeploymentsPanel extends TethysGUIPanel implements DeploymentTableObserver {
+public class DeploymentsPanel extends TethysExportPanel implements DeploymentTableObserver {
 	
-	private JPanel mainPanel;
+//	private JPanel mainPanel;
 	
 	private PAMGuardDeploymentsTable pamDeploymentsTable;
 	
 	private DeploymentExportPanel exportPanel;
 	
-	private JButton exportButton, optionsButton;
-//	private TethysDeploymentsTable tethysDeploymentsTable;
-	private JLabel exportWarning;
+	private JLabel effortName;
 
 	public DeploymentsPanel(TethysControl tethysControl) {
-		super(tethysControl);
+		super(tethysControl, tethysControl.getDeploymentHandler(), true);
 		DeploymentHandler deploymentHandler = tethysControl.getDeploymentHandler();
 		pamDeploymentsTable = new PAMGuardDeploymentsTable(tethysControl);
 		exportPanel = new DeploymentExportPanel(tethysControl, pamDeploymentsTable);
 		pamDeploymentsTable.addObserver(exportPanel);
-//		tethysDeploymentsTable = new TethysDeploymentsTable(tethysControl);
-		mainPanel = new PamPanel(new BorderLayout());
+		
+		JPanel mainPanel = getMainPanel();
 		mainPanel.setBorder(new TitledBorder("Recording periods and deployment information"));
 		pamDeploymentsTable.addObserver(this);
 		pamDeploymentsTable.addObserver(deploymentHandler);
-//		JSplitPane splitPane = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT);
-//		splitPane.add(pamDeploymentsTable.getComponent());
-//		splitPane.add(tethysDeploymentsTable.getComponent());
-//		mainPanel.add(splitPane,BorderLayout.CENTER);
-//		SwingUtilities.invokeLater(new Runnable() {
-//			
-//			@Override
-//			public void run() {
-//				splitPane.setDividerLocation(0.6);
-//			}
-//		});
-		JPanel ctrlPanel = new PamPanel(new BorderLayout());
-		JPanel ctrlButtons = new JPanel();
-		ctrlButtons.setLayout(new BoxLayout(ctrlButtons, BoxLayout.X_AXIS));
-		optionsButton = new JButton("Options ...");
-		exportButton = new JButton("Export ...");
-		ctrlButtons.add(optionsButton);
-		ctrlButtons.add(exportButton);
-		ctrlPanel.add(BorderLayout.WEST, ctrlButtons);
 		
-		optionsButton.addActionListener(new ActionListener() {
-			@Override
-			public void actionPerformed(ActionEvent e) {
-				getTethysControl().getDeploymentHandler().showOptions(null);
-			}
-		});
-		
-		exportButton.addActionListener(new ActionListener() {
-			@Override
-			public void actionPerformed(ActionEvent e) {
-				exportDeployments();
-			}
-		});
-		exportWarning = new JLabel(" ");
-		ctrlPanel.add(BorderLayout.CENTER, exportWarning);
-		
-		mainPanel.add(BorderLayout.CENTER, pamDeploymentsTable.getComponent());
-		mainPanel.add(BorderLayout.NORTH, ctrlPanel);
-//		mainPanel.add(BorderLayout.EAST, exportPanel.getComponent());
-		exportButton.setEnabled(false);
-	}
-
-	protected void exportDeployments() {
-		getTethysControl().getDeploymentHandler().exportDeployments();
-	}
-
-	@Override
-	public JComponent getComponent() {
-		return mainPanel;
+		effortName = new JLabel("   ");
+		JPanel centralPanel = new JPanel(new BorderLayout());
+		centralPanel.add(BorderLayout.NORTH, effortName);
+		centralPanel.add(BorderLayout.CENTER,pamDeploymentsTable.getComponent());
+		mainPanel.add(BorderLayout.CENTER, centralPanel);
 	}
 
 	@Override
@@ -98,12 +55,21 @@ public class DeploymentsPanel extends TethysGUIPanel implements DeploymentTableO
 		enableExportButton();
 	}
 	
-	
-
 	private void enableExportButton() {
+		if (!getTethysControl().isServerOk()) {
+			disableExport("Tethys server not running");
+			return;
+		}
+		
+		CalibrationHandler calHandler = getTethysControl().getCalibrationHandler();
+		if (calHandler.haveAllChannelCalibrations() == false) {
+			disableExport("Calibration data for each channel must be exported before creating Deployment documents");
+			return;
+		}
+		
 		ArrayList<RecordingPeriod> selected = pamDeploymentsTable.getSelectedPeriods();
-		if (selected == null) {
-			exportButton.setEnabled(false);
+		if (selected == null || selected.size() == 0) {
+			disableExport("You must select one or more deployment periods to export");
 			return;
 		}
 		boolean existing = false;
@@ -118,17 +84,35 @@ public class DeploymentsPanel extends TethysGUIPanel implements DeploymentTableO
 		}
 		String warning = null;
 		if (existing) {
-			warning = "  One or more deployment documents already exist. These must be deleted prior to exporting new documents";
-			exportWarning.setText(warning);
+			warning = "One or more deployment documents already exist. These must be deleted prior to exporting new documents";
+			disableExport(warning);
+			return;
 		}
 		
-		exportButton.setEnabled(selected.size()>0 & existing == false && getTethysControl().isServerOk());
+		enableExport(true);
 	}
 
 	@Override
 	public void updateState(TethysState tethysState) {
 		super.updateState(tethysState);
 		enableExportButton();
+		RecordingList recordingList = pamDeploymentsTable.getMasterList();
+		if (recordingList == null) {
+			effortName.setText("  No available effort data");
+		}
+		else {
+			effortName.setText("  Effort from " + recordingList.getSourceName());
+		}
+	}
+
+	@Override
+	protected void exportButtonPressed(ActionEvent e) {
+		getTethysControl().getDeploymentHandler().exportDeployments();
+	}
+
+	@Override
+	protected void optionsButtonPressed(ActionEvent e) {
+		getTethysControl().getDeploymentHandler().showOptions(null);
 	}
 
 

src/tethys/swing/FancyClientButton.java

@@ -1,12 +1,15 @@
 package tethys.swing;
 
 import java.awt.BorderLayout;
+import java.awt.Desktop;
 import java.awt.FlowLayout;
 import java.awt.GridBagConstraints;
 import java.awt.GridBagLayout;
 import java.awt.Insets;
 import java.awt.event.ActionEvent;
 import java.awt.event.ActionListener;
+import java.io.File;
+import java.io.IOException;
 
 import javax.swing.AbstractButton;
 import javax.swing.BoxLayout;
@@ -30,14 +33,14 @@ import tethys.dbxml.DBXMLConnect;
  */
 public class FancyClientButton extends JPanel {
 
+	private TethysControl tethysControl;
+	
 	private JButton clientButton;
 	private JButton dropButton;
 	private JPopupMenu collectionsMenu;
-	private TethysControl tethysControl;
 	private JCheckBoxMenuItem showBrowser;
 	private AbstractButton showPAMGuard;
 
-
 	public FancyClientButton(TethysControl tethysControl) {
 		this.tethysControl = tethysControl;
 		setLayout(new GridBagLayout());
@@ -105,6 +108,16 @@ public class FancyClientButton extends JPanel {
 			menuItem.addActionListener(new OpenCollection(collections[i]));
 			collectionsMenu.add(menuItem);
 		}
+		collectionsMenu.addSeparator();
+		JMenuItem tmpItem = new JMenuItem("Open temp folder");
+		collectionsMenu.add(tmpItem);
+		tmpItem.setToolTipText("Open folder used for temporary document files during export in Windows Explorer");
+		tmpItem.addActionListener(new ActionListener() {
+			@Override
+			public void actionPerformed(ActionEvent e) {
+				openTempFolder();
+			}
+		});
 
 		dropButton.addActionListener(new ActionListener() {
 			@Override
@@ -115,6 +128,19 @@ public class FancyClientButton extends JPanel {
 		enableItems();
 	}
 
+	protected void openTempFolder() {
+		File tempFolder = tethysControl.getDbxmlConnect().checkTempFolder();
+		if (tempFolder == null) {
+			return;
+		}
+		try {
+			Desktop.getDesktop().open(tempFolder);
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+
 	protected void enableItems() {
 		boolean isP = tethysControl.getTethysExportParams().listDocsInPamguard;
 		showBrowser.setSelected(!isP);

src/tethys/swing/PAMGuardDeploymentsTable.java

@@ -32,6 +32,7 @@ import tethys.TethysState.StateType;
 import tethys.dbxml.TethysException;
 import tethys.deployment.DeploymentHandler;
 import tethys.deployment.DeploymentOverview;
+import tethys.deployment.RecordingList;
 import tethys.deployment.RecordingPeriod;
 import tethys.niluswraps.PDeployment;
 
@@ -55,6 +56,8 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 	
 	private ArrayList<DeploymentTableObserver> observers = new ArrayList<>();
 
+private RecordingList masterList;
+
 	public PAMGuardDeploymentsTable(TethysControl tethysControl) {
 		super(tethysControl);
 //		deploymentHandler = new DeploymentHandler(getTethysControl());
@@ -75,6 +78,10 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 		return mainPanel;
 	}
 	
+	public RecordingList getMasterList() {
+		return masterList;
+	}
+
 	private class TableMouse extends MouseAdapter {
 
 		@Override
@@ -95,7 +102,7 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 		public void mouseClicked(MouseEvent e) {
 			int aRow = table.getSelectedRow();
 			int col = table.getSelectedColumn();
-			ArrayList<RecordingPeriod> periods = deploymentOverview.getRecordingPeriods();
+			ArrayList<RecordingPeriod> periods = getMasterList().getEffortPeriods();
 			if (aRow >= 0 && aRow < periods.size() && col == TableModel.SELECTCOLUMN) {
 				periods.get(aRow).toggleSelected();
 				notifyObservers();
@@ -118,7 +125,8 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 		}
 		// make a list of RecordingPeriods which don't currently have a Deployment document
 		ArrayList<RecordingPeriod> newPeriods = new ArrayList<>();
-		ArrayList<RecordingPeriod> allPeriods = deploymentOverview.getRecordingPeriods();
+		ArrayList<RecordingPeriod> allPeriods = getMasterList().getEffortPeriods();
+//		ArrayList<RecordingPeriod> allPeriods = deploymentOverview.getRecordingPeriods();
 		ArrayList<PDeployment> matchedDeployments = new ArrayList<>();
 		for (int i = 0; i < selRows.length; i++) {
 			PDeployment tethysDeployment = allPeriods.get(selRows[i]).getMatchedTethysDeployment();
@@ -200,7 +208,8 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 	}
 	
 	protected void selectAll(boolean select) {
-		ArrayList<RecordingPeriod> recordingPeriods = deploymentOverview.getRecordingPeriods();
+		ArrayList<RecordingPeriod> recordingPeriods = getMasterList().getEffortPeriods();
+//		ArrayList<RecordingPeriod> recordingPeriods = deploymentOverview.getRecordingPeriods();
 		for (int i = 0; i < recordingPeriods.size(); i++) {
 			recordingPeriods.get(i).setSelected(select);
 		}
@@ -329,7 +338,7 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 		if (deploymentOverview == null) {
 			return null;
 		}
-		ArrayList<RecordingPeriod> allPeriods = deploymentOverview.getRecordingPeriods();
+		ArrayList<RecordingPeriod> allPeriods = getMasterList().getEffortPeriods();
 		ArrayList<RecordingPeriod> selPeriods = new ArrayList();
 		int n = allPeriods.size();
 		for (int i = 0; i < n; i++) {
@@ -348,6 +357,7 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 	private void updateDeployments() {
 		DeploymentHandler deploymentHandler = getTethysControl().getDeploymentHandler();
 		deploymentOverview = deploymentHandler.getDeploymentOverview();
+		masterList = deploymentOverview.getMasterList(getTethysControl());
 		if (deploymentOverview == null) {
 			return;
 		}
@@ -373,7 +383,7 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 				return 0;
 			}
 			else {
-				return deploymentOverview.getRecordingPeriods().size();
+				return getMasterList().size();
 			}
 		}
 
@@ -398,13 +408,14 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 
 		@Override
 		public Object getValueAt(int rowIndex, int columnIndex) {
-			RecordingPeriod period = deploymentOverview.getRecordingPeriods().get(rowIndex);
+			RecordingList masterList = getMasterList();
+			RecordingPeriod period = masterList.getEffortPeriods().get(rowIndex);
 //			DeploymentRecoveryPair deplInfo = deploymentInfo.get(rowIndex);
 			if (columnIndex == 6) {
-				return deploymentOverview.getDutyCycleInfo();
+				return masterList.assessDutyCycle();
 			}
 			if (columnIndex == 4 && rowIndex > 0) {
-				RecordingPeriod prevPeriod = deploymentOverview.getRecordingPeriods().get(rowIndex-1);
+				RecordingPeriod prevPeriod = masterList.getEffortPeriods().get(rowIndex-1);
 				long gap = period.getRecordStart() - prevPeriod.getRecordStop();
 				return PamCalendar.formatDuration(gap);
 			}

src/tethys/swing/TethysConnectionPanel.java

@@ -277,12 +277,17 @@ public class TethysConnectionPanel extends TethysGUIPanel {
 	@Override
 	public void updateState(TethysState tethysState) {
 		super.updateState(tethysState);
-		if (tethysState.stateType == StateType.UPDATESERVER) {
+		switch (tethysState.stateType) {
+		case UPDATESERVER:
 			fillServerControl();
 			updateProjectList();
-		}
-		if (tethysState.stateType == StateType.NEWPROJECTSELECTION) {
+			break;
+		case NEWPROJECTSELECTION:
 			updateProjectList();
+			break;
+		case UPDATEMETADATA:
+			updateInstrumentsList();
+			break;
 		}
 	}
 

src/tethys/swing/TethysDeploymentsTable.java

@@ -21,6 +21,7 @@ import tethys.TethysControl;
 import tethys.TethysMenuActions;
 import tethys.TethysState;
 import tethys.deployment.DeploymentOverview;
+import tethys.deployment.RecordingList;
 import tethys.niluswraps.PDeployment;
 
 public class TethysDeploymentsTable extends TethysGUIPanel {
@@ -123,7 +124,6 @@ public class TethysDeploymentsTable extends TethysGUIPanel {
 			return columnNames[column];
 		}		
 
-
 		public String getMatchText(PDeployment deployment) {
 			// TODO Auto-generated method stub
 			if (deployment.getMatchedPAMGaurdPeriod() != null) {
@@ -132,8 +132,9 @@ public class TethysDeploymentsTable extends TethysGUIPanel {
 			if (deploymentOverview == null) {
 				return "No PAMGuard data";
 			}
-			Long depStart = deploymentOverview.getFirstStart();
-			Long depEnd = deploymentOverview.getLastEnd();
+			RecordingList masterList = deploymentOverview.getMasterList(getTethysControl());
+			Long depStart = masterList.getStart();
+			Long depEnd = masterList.getEnd();
 			if (depStart == null) {
 				return "No PAMGuard recordings";
 			}

src/tethys/swing/TethysExportPanel.java

@@ -0,0 +1,227 @@
+package tethys.swing;
+
+import java.awt.BorderLayout;
+import java.awt.FlowLayout;
+import java.awt.event.ActionEvent;
+import java.awt.event.ActionListener;
+
+import javax.swing.BoxLayout;
+import javax.swing.JButton;
+import javax.swing.JComponent;
+import javax.swing.JLabel;
+import javax.swing.JPanel;
+
+import PamView.dialog.NorthPanel;
+import PamView.dialog.SettingsButton;
+import PamView.help.PamHelp;
+import tethys.Collection;
+import tethys.CollectionHandler;
+import tethys.TethysControl;
+
+/**
+ * Common panel used by Calibrations, Deployments and Detections to show an export button and other 
+ * common components, such as help, a table, etc.. 
+ * @author dg50
+ *
+ */
+abstract public class TethysExportPanel extends TethysGUIPanel {
+	
+	private TippedButton exportButton;
+	
+	private JButton optionsButton, helpButton;
+	
+	private JPanel mainPanel, northPanel;
+	
+	private JLabel message;
+
+	private CollectionHandler collectionHandler;
+
+	private String helpPoint;
+
+	private boolean showOptions;
+
+	public TethysExportPanel(TethysControl tethysControl, CollectionHandler collectionHandler, boolean showOptions) {
+		super(tethysControl);
+		this.collectionHandler = collectionHandler;
+		this.showOptions = showOptions;
+		this.helpPoint = collectionHandler.getHelpPoint();
+		
+		mainPanel = new JPanel(new BorderLayout());
+		northPanel = new JPanel(new BorderLayout());
+		JPanel nwPanel = new JPanel();
+		nwPanel.setLayout(new BoxLayout(nwPanel, BoxLayout.X_AXIS));
+		JPanel nePanel = new JPanel();
+		nePanel.setLayout(new BoxLayout(nePanel, BoxLayout.X_AXIS));
+		northPanel.add(BorderLayout.CENTER, nwPanel);
+		northPanel.add(BorderLayout.EAST, nePanel);
+		mainPanel.add(BorderLayout.NORTH, northPanel);
+		
+		optionsButton = new SettingsButton();
+		exportButton = new TippedButton("Export ...", "Export " + collectionHandler.collectionName() + " to Tethys");
+		helpButton = new JButton("?");
+		helpButton.setToolTipText("Show context sensitive help");
+		JLabel space = new JLabel("  ");
+		message = new JLabel (" ");
+		
+		nwPanel.add(optionsButton);
+		nwPanel.add(exportButton);
+		nwPanel.add(space);
+		nwPanel.add(message);
+		nePanel.add(helpButton);
+		
+		showAndHide();
+		
+		optionsButton.addActionListener(new ActionListener() {
+			@Override
+			public void actionPerformed(ActionEvent e) {
+				optionsButtonPressed(e);
+			}
+		});
+		exportButton.addActionListener(new ActionListener() {
+			@Override
+			public void actionPerformed(ActionEvent e) {
+				exportButtonPressed(e);
+			}
+		});
+		helpButton.addActionListener(new ActionListener() {
+			
+			@Override
+			public void actionPerformed(ActionEvent e) {
+				helpButtonPressed(e);
+			}
+		});
+	}
+
+	/**
+	 * Show the help. 
+	 * @param e
+	 */
+	protected void helpButtonPressed(ActionEvent e) {
+		if (helpPoint == null) {
+			return;
+		}
+		PamHelp.getInstance().displayContextSensitiveHelp(helpPoint);
+	}
+
+	/**
+	 * Export button has been pressed. 
+	 * @param e
+	 */
+	protected abstract void exportButtonPressed(ActionEvent e);
+
+	/**
+	 * Options button has been pressed. 
+	 * @param e
+	 */
+	protected abstract void optionsButtonPressed(ActionEvent e);
+
+	private void showAndHide() {
+		optionsButton.setVisible(showOptions);
+		helpButton.setVisible(helpPoint != null);
+	}
+
+	@Override
+	public JComponent getComponent() {
+		return mainPanel;
+	}
+
+	/**
+	 * @return the helpPoint
+	 */
+	public String getHelpPoint() {
+		return helpPoint;
+	}
+
+	/**
+	 * @param helpPoint the helpPoint to set
+	 */
+	public void setHelpPoint(String helpPoint) {
+		this.helpPoint = helpPoint;
+		showAndHide();
+	}
+
+	/**
+	 * @return the showOptions
+	 */
+	public boolean isShowOptions() {
+		return showOptions;
+	}
+
+	/**
+	 * @param showOptions the showOptions to set
+	 */
+	public void setShowOptions(boolean showOptions) {
+		this.showOptions = showOptions;
+		showAndHide();
+	}
+
+	/**
+	 * @return the exportButton
+	 */
+	public TippedButton getExportButton() {
+		return exportButton;
+	}
+
+	/**
+	 * @return the optionsButton
+	 */
+	public JButton getOptionsButton() {
+		return optionsButton;
+	}
+
+	/**
+	 * @return the helpButton
+	 */
+	public JButton getHelpButton() {
+		return helpButton;
+	}
+
+	/**
+	 * @return the mainPanel
+	 */
+	public JPanel getMainPanel() {
+		return mainPanel;
+	}
+
+	/**
+	 * @return the northPanel
+	 */
+	public JPanel getNorthPanel() {
+		return northPanel;
+	}
+
+	/**
+	 * @return the message
+	 */
+	public JLabel getMessage() {
+		return message;
+	}
+
+	/**
+	 * @return the collectionHandler
+	 */
+	public CollectionHandler getCollectionHandler() {
+		return collectionHandler;
+	}
+	
+	/**
+	 * Enable or disable export button, leaving tool tips alone
+	 * @param enable
+	 */
+	public void enableExport(boolean enable) {
+		exportButton.setEnabled(enable);
+		if (enable) {
+			message.setText(null);
+		}
+	}
+	
+	/**
+	 * Disable the export button and set the tooltip. 
+	 * @param disabledTip
+	 */
+	public void disableExport(String disabledTip) {
+		exportButton.disable(disabledTip);
+		message.setText(disabledTip);
+	}
+
+}

src/tethys/swing/TippedButton.java

@@ -0,0 +1,77 @@
+package tethys.swing;
+
+import javax.swing.JButton;
+
+public class TippedButton extends JButton {
+
+	private static final long serialVersionUID = 1L;
+	private String enabledTip;
+	private String disabledTip;
+
+	/**
+	 * Create a button with standard tips which will be used for enabled state
+	 * @param text
+	 * @param enabledTip
+	 */
+	public TippedButton(String text, String enabledTip) {
+		this(text, enabledTip, null);
+	}
+	
+	/**
+	 * Create a button with standard tips which will be used for enabled and disabled state
+	 * @param text
+	 * @param enabledTip
+	 * @param disabledTip
+	 */
+	public TippedButton(String text, String enabledTip, String disabledTip) {
+		super(text);
+		this.enabledTip = enabledTip;
+		this.disabledTip = disabledTip;
+		setToolTipText(enabledTip);
+	}
+
+	@Override
+	public void setEnabled(boolean enable) {
+		super.setEnabled(enable);
+		setToolTipText(enable ? enabledTip : disabledTip);
+	}
+	
+	/*
+	 * Call to disable the button and at the same time 
+	 * set a tooltip giving the reason. 
+	 */
+	public void disable(String newTip) {
+		disabledTip = newTip;
+		setEnabled(false);
+	}
+
+	/**
+	 * @return the enabledTip
+	 */
+	public String getEnabledTip() {
+		return enabledTip;
+	}
+
+	/**
+	 * @param enabledTip the enabledTip to set
+	 */
+	public void setEnabledTip(String enabledTip) {
+		this.enabledTip = enabledTip;
+	}
+
+	/**
+	 * @return the disabledTip
+	 */
+	public String getDisabledTip() {
+		return disabledTip;
+	}
+
+	/**
+	 * @param disabledTip the disabledTip to set
+	 */
+	public void setDisabledTip(String disabledTip) {
+		this.disabledTip = disabledTip;
+	}
+	
+
+}