Merge updates from Tethys branch (#133)

* Tidying

Lots of GUI improvement and code tidying. Functionality to export
gzipped documents to reduce traffic.

* V 2.02.10ba for user testing

* Tidy up click selector

Improve layout and tips on dialog and improve logic for manual and
automatic event types.

* Menu tide up

.classpath

@@ -6,9 +6,8 @@
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER">
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-11">
 		<attributes>
-			<attribute name="module" value="true"/>
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>

dependency-reduced-pom.xml

@@ -4,7 +4,7 @@
   <groupId>org.pamguard</groupId>
   <artifactId>Pamguard</artifactId>
   <name>Pamguard Java12+</name>
-  <version>2.02.10ae</version>
+  <version>2.02.10bb</version>
   <description>Pamguard for Java 12+, using Maven to control dependcies</description>
   <url>www.pamguard.org</url>
   <organization>

pom.xml

@@ -5,7 +5,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.pamguard</groupId>
   <artifactId>Pamguard</artifactId>
-  <version>2.02.10ae</version>
+  <version>2.02.10bb</version>
   <name>Pamguard Java12+</name>
   <description>Pamguard for Java 12+, using Maven to control dependcies</description>
 	<url>www.pamguard.org</url>

src/PamController/DataIntegrityChecker.java

@@ -0,0 +1,12 @@
+package PamController;
+
+/**
+ * Provides a set of functions that can check and repair data. 
+ * @author dg50
+ *
+ */
+public interface DataIntegrityChecker {
+
+	public boolean checkDataStore();
+	
+}

src/PamController/DataOutputStore.java

@@ -29,4 +29,11 @@ public interface DataOutputStore extends OfflineDataStore {
 	 */
 	public boolean deleteDataFrom(long timeMillis);
 	
+	/**
+	 * Get a data integrity checker. This can be called at startup to see if there is  a problem. 
+	 * @return
+	 */
+	public DataIntegrityChecker getInegrityChecker();
+	
+	
 }

src/PamController/PamguardVersionInfo.java

@@ -31,12 +31,12 @@ public class PamguardVersionInfo {
 	 * Version number, major version.minorversion.sub-release.
 	 * Note: can't go higher than sub-release 'f'
 	 */
-	static public final String version = "2.02.10ae";
+	static public final String version = "2.02.10bb";
 	
 	/**
 	 * Release date
 	 */
-	static public final String date = "15 March 2024";
+	static public final String date = "22 March 2024";
 	
 //	/**
 //	 * Release type - Beta or Core

src/PamUtils/worker/PamWorkDialog.java

@@ -1,5 +1,7 @@
 package PamUtils.worker;
 
+import java.awt.Dimension;
+import java.awt.Point;
 import java.awt.Window;
 
 import javax.swing.BoxLayout;
@@ -23,6 +25,9 @@ public class PamWorkDialog extends PamDialog {
 		mainPanel.setBorder(new TitledBorder("Task Progress"));
 //		GridBagConstraints c = new PamGridBagContraints();
 		mainPanel.add(progressBar = new JProgressBar(0, 100));
+		Dimension sz = progressBar.getPreferredSize();
+		sz.width = 300;
+		progressBar.setPreferredSize(sz);
 		textRows = new PamTextDisplay[nTextRows];
 		for (int i = 0; i < nTextRows; i++) {
 //			c.gridy++;
@@ -35,6 +40,18 @@ public class PamWorkDialog extends PamDialog {
 		getButtonPanel().setVisible(false);
 		setDialogComponent(mainPanel);
 		setResizable(true);
+		
+		if (parentFrame != null) {
+			Dimension prefSize = this.getPreferredSize();
+			Point screenLoc = parentFrame.getLocationOnScreen();
+			int x = (parentFrame.getWidth()-prefSize.width)/2;
+			int y = (parentFrame.getHeight()-prefSize.height)/2;
+			if (screenLoc != null) {
+				x += screenLoc.x;
+				y += screenLoc.y;
+			}
+			setLocation(x, y);
+		}
 	}
 	
 	public void update(PamWorkProgressMessage progressMsg) {

src/PamView/panel/SeparatorBorder.java

@@ -0,0 +1,27 @@
+package PamView.panel;
+
+import java.awt.Component;
+import java.awt.FontMetrics;
+import java.awt.Graphics;
+
+import javax.swing.border.TitledBorder;
+
+public class SeparatorBorder extends TitledBorder {
+
+	private static final long serialVersionUID = 1L;
+
+	public SeparatorBorder(String title) {
+		super(title);
+		// TODO Auto-generated constructor stub
+	}
+	 public void paintBorder(Component c, Graphics g, int x, int y, int width, int height) {
+		 FontMetrics fm = g.getFontMetrics();
+		 if (fm != null) {
+			 height = fm.getHeight()-EDGE_SPACING*2-1;
+		 }
+		 else {
+			 height = 12;
+		 }
+		super.paintBorder(c, g, x, y, width, height);   
+	 }
+}

src/RightWhaleEdgeDetector/species/RWTethysDataProvider.java

@@ -29,6 +29,7 @@ public class RWTethysDataProvider extends AutoTethysProvider {
 		Parameters parameters = detection.getParameters();
 		parameters.setScore((double) rweDataUnit.rweSound.soundType);
 		double snr = 20.*Math.log10(rweDataUnit.rweSound.signal/rweDataUnit.rweSound.noise);
+		snr = AutoTethysProvider.roundDecimalPlaces(snr, 1);
 		parameters.setSNRDB(snr);
 		
 		return detection;

src/binaryFileStorage/BinaryStore.java

@@ -37,6 +37,7 @@ import dataMap.OfflineDataMap;
 import dataMap.OfflineDataMapPoint;
 import PamController.AWTScheduler;
 import PamController.DataInputStore;
+import PamController.DataIntegrityChecker;
 import PamController.DataOutputStore;
 import PamController.OfflineDataStore;
 import PamController.PamControlledUnit;
@@ -74,6 +75,7 @@ import annotation.binary.AnnotationBinaryData;
 import annotation.handler.AnnotationHandler;
 import backupmanager.BackupInformation;
 import binaryFileStorage.backup.BinaryBackupStream;
+import binaryFileStorage.checker.BinaryIntegrityChecker;
 import binaryFileStorage.layoutFX.BinaryStoreGUIFX;
 
 /**
@@ -946,6 +948,8 @@ PamSettingsSource, DataOutputStore {
 			PamController.getInstance().notifyModelChanged(PamControllerInterface.CHANGED_OFFLINE_DATASTORE);
 //			System.out.println("BinaryDataMapMaker really done " + this);
 			dataMapMaker = null;
+			
+			getInegrityChecker().checkDataStore();
 		}
 
 		@Override
@@ -2593,5 +2597,9 @@ PamSettingsSource, DataOutputStore {
 	public String getDataLocation() {
 		return binaryStoreSettings.getStoreLocation();
 	}
+	@Override
+	public DataIntegrityChecker getInegrityChecker() {
+		return new BinaryIntegrityChecker(this);
+	}
 	
 }

src/binaryFileStorage/checker/BinaryIntegrityChecker.java

@@ -0,0 +1,54 @@
+package binaryFileStorage.checker;
+
+import java.util.ArrayList;
+
+import PamController.DataIntegrityChecker;
+import PamController.PamController;
+import PamView.dialog.warn.WarnOnce;
+import PamguardMVC.PamDataBlock;
+import binaryFileStorage.BinaryStore;
+import dataMap.OfflineDataMap;
+
+public class BinaryIntegrityChecker implements DataIntegrityChecker {
+
+	private BinaryStore binaryStore;
+
+	public BinaryIntegrityChecker(BinaryStore binaryStore) {
+		this.binaryStore = binaryStore;
+	}
+
+	public boolean checkDataStore() {
+		checkMapOverlaps();
+		
+		
+		return true;
+	}
+
+	private boolean checkMapOverlaps() {
+		boolean ok = true;
+		ArrayList<PamDataBlock> dataBlocks = binaryStore.getStreamingDataBlocks(false);
+		for (PamDataBlock aBlock : dataBlocks) {
+			ok &= checkMapOverlaps(aBlock);
+		}
+		return ok;
+	}
+
+	private boolean checkMapOverlaps(PamDataBlock aBlock) {
+		OfflineDataMap dataMap = aBlock.getOfflineDataMap(binaryStore);
+		if (dataMap == null) {
+			return true;
+		}
+		long overlaps = dataMap.checkOverlaps();
+		if (overlaps <= 0) {
+			return true;
+		}
+		String warn = String.format("<html>Binary data %s has overlapping data files, with a maximum overlap of %3.1fs<br>"
+				+ "This can occur when data have been reprocessed multiple times offline into "
+				+ "the same folders.<br>Since files sometimes get slightly different names, old files are"
+				+ "not always overwritten. <br>"
+				+ "You should determine which files are 'old' by looking at file creation dates and delete them.",
+				aBlock.getLongDataName(), (double) overlaps / 1000.);
+		WarnOnce.showNamedWarning("BINOVERLAPWARNING", PamController.getMainFrame(), "Data Integrity Warning", warn, WarnOnce.WARNING_MESSAGE);
+		return false;
+	}
+}

src/clickDetector/dataSelector/ClickDataSelector.java

@@ -112,14 +112,23 @@ public class ClickDataSelector extends DataSelector {
 		
 		// see if there is a super detection and see if it's got a comment. 
 		String comment = oev.getComment();
+		
+		boolean isAutomatic = false;
 		if (comment != null) {
-			if (clickAlarmParameters.onlineAutoEvents && comment.startsWith("Automatic")) {
-				return true;
-			}
-			if (clickAlarmParameters.onlineManualEvents && comment.startsWith("Manual")) {
-				return true;
-			}
+			isAutomatic = comment.startsWith("Automatic");
 		}
+		if (isAutomatic && clickAlarmParameters.onlineAutoEvents) {
+			return true;
+		}
+		else if (clickAlarmParameters.onlineManualEvents) {
+			return true;
+		}
+//			if (clickAlarmParameters.onlineAutoEvents && comment.startsWith("Automatic")) {
+//				return true;
+//			}
+//			if (clickAlarmParameters.onlineManualEvents && comment.startsWith("Manual")) {
+//				return true;
+//			}
 		/*
 		 * Otherwise need to work out where the hell the event type is in the 
 		 * list of event types and see if it's wanted. 

src/clickDetector/dataSelector/ClickSelectPanel.java

@@ -30,6 +30,8 @@ import PamView.dialog.PamDialog;
 import PamView.dialog.PamDialogPanel;
 import PamView.dialog.PamGridBagContraints;
 import PamView.panel.PamAlignmentPanel;
+import PamView.panel.SeparatorBorder;
+import PamView.panel.WestAlignedPanel;
 
 public class ClickSelectPanel implements PamDialogPanel {
 	
@@ -40,6 +42,8 @@ public class ClickSelectPanel implements PamDialogPanel {
 	private ClickDataSelector clickDataSelector;
 	private JPanel mainPanel;
 	private boolean isViewer;
+	
+	public static final String mainTip = "You should select options in both the Click Type and the Event Type panels";
 
 	public ClickSelectPanel(ClickDataSelector clickDataSelector, boolean allowScores, boolean useEventTypes) {
 		this.clickDataSelector = clickDataSelector;
@@ -79,7 +83,9 @@ public class ClickSelectPanel implements PamDialogPanel {
 		private LookupList lutList;
 		
 		public EventTypePanel() {
-			setBorder(new TitledBorder("Event Type Selection"));
+			setBorder(new SeparatorBorder("Event Type Selection"));
+
+			setToolTipText(mainTip);
 		}
 		
 		void setParams() {
@@ -102,16 +108,24 @@ public class ClickSelectPanel implements PamDialogPanel {
 			boxPanel.add(onlineAuto = new JCheckBox("Automatically detected click trains"), c);
 			onlineAuto.setSelected(clickAlarmParameters.onlineAutoEvents);
 			
+			useUnassigned.setToolTipText("Clicks that are NOT part of a manual or automatic click train");
+			onlineManual.setToolTipText("Clicks that are part of a manually marked click train");
+			onlineAuto.setToolTipText("Clicks that are part of an automatically detected click train");
+			
 			
 			lutList = LookUpTables.getLookUpTables().getLookupList(ClicksOffline.ClickTypeLookupName);
 			if (lutList == null) {
 				return;
 			}
+			c.gridy++;
+			boxPanel.add(new JLabel("OR the following click train types ...", JLabel.LEFT), c);
 			useType = new JCheckBox[lutList.getList().size()];
 			for (int i = 0; i < useType.length; i++) {
 				c.gridy++;
 				boxPanel.add(useType[i] = new JCheckBox(lutList.getList().get(i).getText()), c);
 				useType[i].setSelected(clickAlarmParameters.isUseEventType(lutList.getList().get(i).getCode()));
+				String tip = String.format("Clicks that are part of a click train labelled as %s", lutList.getList().get(i).getText());
+				useType[i].setToolTipText(tip);
 			}
 		}
 		
@@ -158,10 +172,9 @@ public class ClickSelectPanel implements PamDialogPanel {
 			northPanel = new JPanel();
 			northPanel.setLayout(new GridBagLayout());
 			GridBagConstraints c = new PamGridBagContraints();
-			northPanel.setBorder(new TitledBorder("Echoes"));
 			c.gridwidth = 3;
 			c.anchor = GridBagConstraints.WEST;
-			northPanel.add(useEchoes = new JCheckBox("Use Echoes"), c);
+			northPanel.add(new PamAlignmentPanel(useEchoes = new JCheckBox("Use Echoes"), BorderLayout.WEST), c);
 			c.gridwidth = 1;
 			c.gridy++;
 			c.gridx = 0;
@@ -178,11 +191,13 @@ public class ClickSelectPanel implements PamDialogPanel {
 			northPanel.add(scoreByAmplitude = new JCheckBox("Score by amplitude"), c);
 			scoreByAmplitude.setVisible(allowScores);
 			scoreByAmplitude.addActionListener(new AllSpeciesListener());
-			add(BorderLayout.NORTH, northPanel);
+			WestAlignedPanel walpn;
+			add(BorderLayout.NORTH, walpn = new WestAlignedPanel(northPanel));
+			walpn.setBorder(new SeparatorBorder("Echoes"));
 			
 			JPanel centralOuterPanel = new JPanel(new BorderLayout());
 			centralPanel.setLayout(new GridBagLayout());
-			centralOuterPanel.setBorder(new TitledBorder("Click Type Selection"));
+			centralOuterPanel.setBorder(new SeparatorBorder("Click Type Selection"));
 			
 			add(BorderLayout.CENTER, centralOuterPanel);
 			JScrollPane scrollPane = new DialogScrollPane(new PamAlignmentPanel(centralPanel, BorderLayout.WEST), 10);
@@ -197,6 +212,9 @@ public class ClickSelectPanel implements PamDialogPanel {
 			clearAll.addActionListener(new AutoSelect(false));
 			centralOuterPanel.add(BorderLayout.SOUTH, new PamAlignmentPanel(centralEastPanel, BorderLayout.WEST));
 			
+			centralOuterPanel.setToolTipText(mainTip);
+			
+			setToolTipText(mainTip);
 		}
 		
 		void setParams() {

src/dataMap/OfflineDataMap.java

@@ -844,4 +844,23 @@ abstract public class OfflineDataMap<TmapPoint extends OfflineDataMapPoint> {
 			obs.updateDataMap(this, mapPoint);
 		}
 	}
+
+	/**
+	 * Check to see if any data maps overlap in time. 
+	 * @return largest overlap between maps. 0 or -ve means no overlap. 
+	 */
+	public long checkOverlaps() {
+		long bigLap = Long.MIN_VALUE;
+		TmapPoint prevPoint = null;
+		for (TmapPoint mapPoint : mapPoints) {
+			if (prevPoint != null) {
+				long olap = prevPoint.getEndTime() - mapPoint.getStartTime();
+//				if (mapPoint.getStartTime() < prevPoint.getEndTime()) {
+					bigLap = Math.max(olap, bigLap);
+//				}
+			}
+			prevPoint = mapPoint;
+		}
+		return bigLap;
+	}
 }

src/generalDatabase/DBControlUnit.java

@@ -18,6 +18,7 @@ import generalDatabase.backup.DatabaseBackupStream;
 import pamScrollSystem.ViewLoadObserver;
 import pamViewFX.pamTask.PamTaskUpdate;
 import PamController.AWTScheduler;
+import PamController.DataIntegrityChecker;
 import PamController.DataOutputStore;
 import PamController.OfflineDataStore;
 import PamController.PamConfiguration;
@@ -524,5 +525,10 @@ public class DBControlUnit extends DBControl implements DataOutputStore {
 		}
 		return state;
 	}
+	@Override
+	public DataIntegrityChecker getInegrityChecker() {
+		// TODO Auto-generated method stub
+		return null;
+	}
 
 }

src/tethys/CollectionHandler.java

@@ -0,0 +1,38 @@
+package tethys;
+
+abstract public class CollectionHandler {
+
+	private Collection collection;
+	
+	protected TethysControl tethysControl;
+
+	/**
+	 * @param tethysControl
+	 * @param collection
+	 */
+	public CollectionHandler(TethysControl tethysControl, Collection collection) {
+		this.tethysControl = tethysControl;
+		this.collection = collection;
+	}
+	
+	public String collectionName() {
+		return collection.collectionName();
+	}
+
+	/**
+	 * @return the collection
+	 */
+	public Collection getCollection() {
+		return collection;
+	}
+
+	/**
+	 * @return the tethysControl
+	 */
+	public TethysControl getTethysControl() {
+		return tethysControl;
+	}
+	
+	public abstract String getHelpPoint();
+	
+}

src/tethys/TethysControl.java

@@ -32,6 +32,8 @@ import PamController.PamSettingManager;
 import PamController.PamSettings;
 import PamUtils.PamFileChooser;
 import PamUtils.PamFileFilter;
+import PamUtils.worker.PamWorkWrapper;
+import PamUtils.worker.PamWorker;
 import PamView.PamTabPanel;
 import PamView.dialog.warn.WarnOnce;
 import PamguardMVC.PamDataBlock;
@@ -104,7 +106,9 @@ public class TethysControl extends PamControlledUnit implements PamSettings, Tet
 		serverCheckTimer = new Timer(10000, new ActionListener() {
 			@Override
 			public void actionPerformed(ActionEvent e) {
-				checkServer();
+				ServerStatus serverState = checkServer();
+				// check less often when it's OK to generate less debug output. 
+				serverCheckTimer.setDelay(serverState.ok ? 600000 : 5000);
 			}
 		});
 		serverCheckTimer.setInitialDelay(0);
@@ -469,6 +473,8 @@ public class TethysControl extends PamControlledUnit implements PamSettings, Tet
 //		case PamControllerInterface.INITIALIZATION_COMPLETE:
 			initializationStuff();
 			break;
+		case PamControllerInterface.HYDROPHONE_ARRAY_CHANGED:
+			sendStateUpdate(new TethysState(StateType.UPDATEMETADATA));
 		}
 	}
 
@@ -525,7 +531,32 @@ public class TethysControl extends PamControlledUnit implements PamSettings, Tet
 		}
 	}
 
-	private void countProjectDetections() {
+//	/**
+//	 * Count project detections in a worker thread;
+//	 */
+//	public void countProjectDetectionsT() {
+//		sendStateUpdate(new TethysState(StateType.EXPORTRDATA, Collection.Detections));
+//		PamWorker<String> worker = new PamWorker<String>(new PamWorkWrapper<String>() {
+//	
+//			@Override
+//			public String runBackgroundTask(PamWorker<String> pamWorker) {
+//				countProjectDetections();
+//				return null;
+//			}
+//	
+//			@Override
+//			public void taskFinished(String result) {
+//				sendStateUpdate(new TethysState(StateType.NEWPAMGUARDSELECTION, Collection.Detections));
+//	
+//			}
+//		}, getGuiFrame(), 0, "Updating detections counts");
+//		worker.start();
+//	}
+
+	/**
+	 * Count project detections in the current thread. 
+	 */
+	private synchronized void countProjectDetections() {
 		if (dataBlockSynchInfos == null) {
 			return;
 		}
@@ -549,12 +580,6 @@ public class TethysControl extends PamControlledUnit implements PamSettings, Tet
 			
 			i++;
 		}
-//		int[] counts = dbxmlQueries.countDataForProject(deplData.getProject(), dataPrefixes);
-//		if (counts != null) {
-//			for ( i = 0; i < counts.length; i++ ) {
-//				dataBlockSynchInfos.get(i).setDataCount(counts[i]);
-//			}
-//		}
 	}
 
 	/**
@@ -688,11 +713,9 @@ public class TethysControl extends PamControlledUnit implements PamSettings, Tet
 	 * @param dataBlock
 	 */
 	public void exportedDetections(PamDataBlock dataBlock) {
-		sendStateUpdate(new TethysState(StateType.EXPORTRDATA, Collection.Detections));
 		countProjectDetections();
-		sendStateUpdate(new TethysState(StateType.NEWPAMGUARDSELECTION, Collection.Detections));
+//		sendStateUpdate(new TethysState(StateType.NEWPAMGUARDSELECTION, Collection.Detections));
 	}
-
 	/**
 	 * @return the calibrationHandler
 	 */

src/tethys/TethysTimeFuncs.java

@@ -12,6 +12,7 @@ import javax.xml.datatype.DatatypeFactory;
 import javax.xml.datatype.XMLGregorianCalendar;
 
 import PamUtils.PamCalendar;
+import nilus.Helper;
 
 public class TethysTimeFuncs {
 
@@ -54,6 +55,18 @@ public class TethysTimeFuncs {
 	 * @return
 	 */
 	public static XMLGregorianCalendar fromGregorianXML(String gregorianString) {
+		
+		try {
+			XMLGregorianCalendar xmlCal = Helper.timestamp(gregorianString);
+			return xmlCal;
+		} catch (DatatypeConfigurationException e1) {
+			// TODO Auto-generated catch block
+//			e1.printStackTrace();
+		}
+		/**
+		 * Above should work just fine. If it doesn't use my own code below...
+		 */
+		
 		// typical string is 2018-10-20T00:00:00Z
 		if (gregorianString == null) {
 			return null;

src/tethys/calibration/CalibrationHandler.java

@@ -1,6 +1,5 @@
 package tethys.calibration;
 
-import java.lang.reflect.Field;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -37,6 +36,7 @@ import nilus.MetadataInfo;
 import nilus.QualityValueBasic;
 import nilus.ResponsibleParty;
 import tethys.Collection;
+import tethys.CollectionHandler;
 import tethys.DocumentInfo;
 import tethys.DocumentNilusObject;
 import tethys.TethysControl;
@@ -52,9 +52,7 @@ import tethys.niluswraps.NilusUnpacker;
 import tethys.pamdata.AutoTethysProvider;
 import tethys.reporter.TethysReporter;
 
-public class CalibrationHandler implements TethysStateObserver {
-
-	private TethysControl tethysControl;
+public class CalibrationHandler extends CollectionHandler implements TethysStateObserver {
 	
 	private ArrayList<DocumentNilusObject<Calibration>> calibrationsList;
 	
@@ -65,10 +63,14 @@ public class CalibrationHandler implements TethysStateObserver {
 	public static final String[] qaTypes = {"unverified", "valid", "invalid"};
 	
 	private Helper nilusHelper;
+
+	public static final String helpPoint = "utilities.tethys.docs.calibrations";
+	
 	/**
 	 * @param tethysControl
 	 */
 	public CalibrationHandler(TethysControl tethysControl) {
+		super(tethysControl, Collection.Calibrations);
 		this.tethysControl = tethysControl;
 		calibrationsList = new ArrayList();
 		tethysControl.addStateObserver(this);		try {
@@ -504,7 +506,7 @@ public class CalibrationHandler implements TethysStateObserver {
 		}
 		String seachPattern = makeChannelNamePart(iChan);
 		for (int i = 0; i < calibrationsList.size(); i++) {
-			String docName = calibrationsList.get(i).getDocumentName();
+			String docName = calibrationsList.get(i).getDocumentId();
 			if (docName.endsWith(seachPattern)) {
 				return true;
 			}
@@ -570,4 +572,9 @@ public class CalibrationHandler implements TethysStateObserver {
 		}
 		return theseCals;
 	}
+
+	@Override
+	public String getHelpPoint() {
+		return helpPoint;
+	}
 }

src/tethys/calibration/swing/CalibrationsContactCard.java

@@ -5,6 +5,7 @@ import java.awt.GridBagConstraints;
 import java.awt.GridBagLayout;
 import java.awt.event.ActionEvent;
 import java.awt.event.ActionListener;
+import java.util.ArrayList;
 import java.util.Date;
 
 import javax.swing.BoxLayout;
@@ -18,6 +19,9 @@ import javax.xml.datatype.XMLGregorianCalendar;
 
 import org.jdesktop.swingx.JXDatePicker;
 
+import Array.ArrayManager;
+import Array.Hydrophone;
+import Array.PamArray;
 import PamView.dialog.PamDialog;
 import PamView.dialog.PamGridBagContraints;
 import PamView.panel.WestAlignedPanel;
@@ -43,6 +47,8 @@ public class CalibrationsContactCard extends CalibrationsCard {
 	
 	private JButton copyDown, copyUp;
 	
+	private JComboBox<String> hydrophoneSelection;
+	
 	public CalibrationsContactCard(PamWizard pamWizard) {
 		super(pamWizard, "Contact Details");
 		// TODO Auto-generated constructor stub
@@ -57,17 +63,26 @@ public class CalibrationsContactCard extends CalibrationsCard {
 		
 		JPanel datePanel = new JPanel(new GridBagLayout());
 		JPanel lp = new WestAlignedPanel(datePanel);
-		lp.setBorder(new TitledBorder("Calibration date"));
+		lp.setBorder(new TitledBorder("Date and hydrophones"));
 		GridBagConstraints c = new PamGridBagContraints();
 		datePanel.add(new JLabel("Calibration date: ", JLabel.RIGHT), c);
 		datePicker = new JXDatePicker();
 		c.gridx++;
 		datePanel.add(datePicker, c);
-		c.gridx = 0;
-		c.gridy++;
-		datePanel.add(new JLabel("Update Frequency", JLabel.RIGHT), c);
+//		c.gridx = 0;
+		c.gridx++;
+		datePanel.add(new JLabel("  Update Frequency ", JLabel.RIGHT), c);
 		c.gridx++;
 		datePanel.add(updateInterval, c);
+		c.gridx = 0;
+		c.gridy++;
+		datePanel.add(new JLabel("  Hydrophones ", JLabel.RIGHT), c);
+		c.gridwidth = 5;
+		c.gridx++;
+		hydrophoneSelection = new JComboBox<>();
+		datePanel.add(hydrophoneSelection, c);
+		hydrophoneSelection.setToolTipText("Select which hydrophone calibrations to export");
+		
 		
 		calibrator = new ResponsiblePartyPanel("Technical Person");
 		dataManager = new ResponsiblePartyPanel("Data Manager");
@@ -202,6 +217,16 @@ public class CalibrationsContactCard extends CalibrationsCard {
 			datePicker.setDate(new Date(TethysTimeFuncs.millisFromGregorianXML(ts)));
 		}
 		
+		hydrophoneSelection.removeAllItems();
+		hydrophoneSelection.addItem("All hydrophones");
+		PamArray array = ArrayManager.getArrayManager().getCurrentArray();
+		ArrayList<Hydrophone> phones = array.getHydrophoneArray();
+		int i = 0;
+		for (Hydrophone phone : phones) {
+			String txt = String.format("Hydrophone %d, %s, %3.1f dBre1\u00B5Pa", i, phone.getType(), phone.getSensitivity());
+			hydrophoneSelection.addItem(txt);
+			i++;
+		}
 
 	}
 

src/tethys/calibration/swing/CalibrationsExportWizard.java

@@ -5,6 +5,7 @@ import java.awt.Window;
 import PamView.wizard.PamWizard;
 import PamView.wizard.PamWizardCard;
 import nilus.Calibration;
+import tethys.calibration.CalibrationHandler;
 
 public class CalibrationsExportWizard extends PamWizard {
 
@@ -13,8 +14,9 @@ public class CalibrationsExportWizard extends PamWizard {
 	private CalibrationsExportWizard(Window parentFrame, Calibration sampleDocument) {
 		super(parentFrame, "Calibrations Export");
 		this.sampleDocument = sampleDocument;
-		addCard(new CalibrationProcessCard(this));
 		addCard(new CalibrationsContactCard(this));
+		addCard(new CalibrationProcessCard(this));
+		setHelpPoint(CalibrationHandler.helpPoint);
 	}
 	
 	public static Calibration showWizard(Window parentFrame, Calibration sampleDocument) {

src/tethys/calibration/swing/CalibrationsMainPanel.java

@@ -15,53 +15,47 @@ import PamView.panel.PamPanel;
 import tethys.TethysControl;
 import tethys.TethysState;
 import tethys.calibration.CalibrationHandler;
+import tethys.deployment.PInstrument;
+import tethys.swing.TethysExportPanel;
 import tethys.swing.TethysGUIPanel;
+import tethys.swing.TippedButton;
 
-public class CalibrationsMainPanel extends TethysGUIPanel {
+public class CalibrationsMainPanel extends TethysExportPanel {
 
 	private CalibrationHandler calibrationHandler;
 	
 	private CalibrationsTable calibrationsTable;
 	
-	private JPanel mainPanel;
-	
-	private JPanel ctrlPanel;
-	
-	private JButton exportButton;
-	
-	private JLabel warning;
+//	private JPanel mainPanel;
+//	
+//	private JPanel ctrlPanel;
+//	
+//	private TippedButton exportButton;
+//	
+//	private JLabel warning;
 
 	public CalibrationsMainPanel(TethysControl tethysControl, CalibrationHandler calibrationHandler) {
-		super(tethysControl);
+		super(tethysControl, calibrationHandler, false);
 		this.calibrationHandler = calibrationHandler;
-		mainPanel = new PamPanel(new BorderLayout());
+		JPanel mainPanel = getMainPanel();
+//		mainPanel = new PamPanel(new BorderLayout());
 		mainPanel.setBorder(new TitledBorder("Instrument calibration information"));
 		
 		calibrationsTable = new CalibrationsTable(tethysControl, calibrationHandler);
 		mainPanel.add(BorderLayout.CENTER, calibrationsTable.getComponent());
 		
-		ctrlPanel = new PamPanel(new BorderLayout());
-		exportButton = new JButton("Export ...");
-		ctrlPanel.add(BorderLayout.WEST, exportButton);
-		warning = new JLabel();
-		ctrlPanel.add(BorderLayout.CENTER, warning);
-		mainPanel.add(BorderLayout.NORTH, ctrlPanel);
-		exportButton.setToolTipText("Export calibration data to database");
-		exportButton.addActionListener(new ActionListener() {
-			@Override
-			public void actionPerformed(ActionEvent e) {
-				exportCalibrations();
-			}
-		});
-	}
-
-	protected void exportCalibrations() {
-		calibrationHandler.exportAllCalibrations();
-	}
-
-	@Override
-	public JComponent getComponent() {
-		return mainPanel;
+//		ctrlPanel = new PamPanel(new BorderLayout());
+//		exportButton = new TippedButton("Export ...", "Export calibration data to database");
+//		ctrlPanel.add(BorderLayout.WEST, exportButton);
+//		warning = new JLabel();
+//		ctrlPanel.add(BorderLayout.CENTER, warning);
+//		mainPanel.add(BorderLayout.NORTH, ctrlPanel);
+//		exportButton.addActionListener(new ActionListener() {
+//			@Override
+//			public void actionPerformed(ActionEvent e) {
+//				exportCalibrations();
+//			}
+//		});
 	}
 
 	@Override
@@ -71,7 +65,45 @@ public class CalibrationsMainPanel extends TethysGUIPanel {
 	}
 
 	private void enableControls() {
-		exportButton.setEnabled(getTethysControl().isServerOk());
+		if (getTethysControl().isServerOk() == false) {
+			disableExport("Tethys Server not running");
+			return;
+		}
+		if (isHydrophoneNamed() == false) {
+			disableExport("Can't export calibrations until the Hydrophone array has been correctly named");
+			return;
+		};
+		enableExport(true);
+	}
+
+	/**
+	 * Check to see if hydrophone is named correctly. 
+	 * @return
+	 */
+	private boolean isHydrophoneNamed() {
+		PInstrument currentInstrument = getTethysControl().getDeploymentHandler().getCurrentArrayInstrument();
+		if (currentInstrument == null) {
+			return false;
+		}
+		if (currentInstrument.instrumentId == null || currentInstrument.instrumentType == null) {
+			return false;
+		}
+
+		if (currentInstrument.instrumentId.length() == 0 || currentInstrument.instrumentType.length() == 0) {
+			return false;
+		}
+		return true;
+	}
+
+	@Override
+	protected void exportButtonPressed(ActionEvent e) {
+		calibrationHandler.exportAllCalibrations();
+	}
+
+	@Override
+	protected void optionsButtonPressed(ActionEvent e) {
+		// TODO Auto-generated method stub
+		
 	}
 
 }

src/tethys/calibration/swing/CalibrationsTable.java

@@ -120,6 +120,7 @@ public class CalibrationsTable extends TethysGUIPanel {
 			});
 			popMenu.add(menuItem);
 		}
+		popMenu.addSeparator();
 		if (n > 1) {
 			menuItem = new JMenuItem("Delete selected documents");
 			menuItem.addActionListener(new ActionListener() {

src/tethys/dbxml/DBXMLConnect.java

@@ -3,14 +3,17 @@ package tethys.dbxml;
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.io.InputStream;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
-import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.zip.GZIPOutputStream;
 
 import javax.xml.bind.JAXBException;
 
@@ -67,7 +70,7 @@ public class DBXMLConnect {
 		}
 		return true;
 	}
-	
+
 
 	/**
 	 * Get the client. The client will only be recreated if the url changes
@@ -113,9 +116,9 @@ public class DBXMLConnect {
 		}
 		return file;
 	}
-	
+
 	/**
-	 * Create a temporary nilus file. 
+	 * Create a temporary nilus file.
 	 * @param nilusObject
 	 * @return
 	 * @throws TethysException
@@ -124,20 +127,20 @@ public class DBXMLConnect {
 		String tempName = getTempFileName(nilusObject);
 		tempName = tempDirectory.getAbsolutePath() + File.separator + tempName + ".xml";
 		File tempFile = new File(tempName);
-		File retFile = createXMLDocument(nilusObject, tempFile);	
+		File retFile = createXMLDocument(nilusObject, tempFile);
 		retFile.deleteOnExit();
 		return retFile;
 	}
-	
-	
+
+
 	public boolean postAndLog(Object nilusObject) throws TethysException
 	{
 		return postAndLog(nilusObject, null);
 	}
-	
+
 	/**
 	 * I don't think this should ever be used since everything goes a bit pear
-	 * shaped if the documentName isn't the same as the Id. However, for Calibration 
+	 * shaped if the documentName isn't the same as the Id. However, for Calibration
 	 * documents this is no longer the case, since a Calibration can have multiple
 	 * entries on different dates, so allow it !
 	 * @param nilusObject
@@ -145,11 +148,11 @@ public class DBXMLConnect {
 	 * @return
 	 * @throws TethysException
 	 */
-	public boolean postAndLog(Object nilusObject, String documentName) throws TethysException 
-	{	
+	public boolean postAndLog(Object nilusObject, String documentName) throws TethysException
+	{
 		boolean ok = NilusChecker.warnEmptyFields(tethysControl.getGuiFrame(), nilusObject);
-		
-		
+
+
 		TethysException e = null;
 		boolean success = false;
 		try {
@@ -168,13 +171,13 @@ public class DBXMLConnect {
 		}
 		return success;
 	}
-	
+
 	/**
 	 * take a nilus object loaded with PamGuard data and post it to the Tethys database
 	 *
 	 * @param pamGuardObjs a nilus object loaded with PamGuard data
 	 * @return error string, null string means there are no errors
-	 * @throws TethysException 
+	 * @throws TethysException
 	 */
 	private boolean postToTethys(Object nilusObject, String documentName) throws TethysException
 	{
@@ -192,12 +195,35 @@ public class DBXMLConnect {
 			MarshalXML marshal = new MarshalXML();
 			marshal.createInstance(objClass);
 			marshal.marshal(nilusObject, tempFile.toString());
+			// above lines have made a file. Are now going to gzip it before sending to Tethys
+			File zipFile = null;
+			try {
+				zipFile = zipOutputFile(tempFile);
+			}
+			catch (FileNotFoundException e1){
+				System.out.println(e1.getMessage());
+			}
+			catch (IOException e2) {
+				System.out.println(e2.getMessage());
+			}
+			String finalName;
+			if (zipFile == null) {
+				finalName = bodgeName;
+			}
+			else {
+				finalName = zipFile.toString();
+			}
+
+
 			//				tempFile = stripXMLHeader(tempFile);
 			importReturn = Importer.ImportFiles(params.getFullServerName(), collection.collectionName(),
-					new String[] { bodgeName }, "", "", false);
+					new String[] { finalName }, "", "", false);
 
 
 			tempFile.deleteOnExit();
+			if (zipFile != null) {
+				zipFile.deleteOnExit();
+			}
 		} catch(IllegalArgumentException e) {
 			throw new TethysException("IllegalArgumentException posting to Tethys: " + e.getMessage(), null);
 		} catch (IOException e) {
@@ -207,21 +233,49 @@ public class DBXMLConnect {
 		}
 
 		/*
-		 * The returned string consists of the file name, then an XML report. 
-		 * Quite hard to see much common structure in this, so just look for 
+		 * The returned string consists of the file name, then an XML report.
+		 * Quite hard to see much common structure in this, so just look for
 		 * two words, <Success> and <Error>
 		 */
 		boolean error = importReturn.contains("<Error");
 		boolean success = importReturn.contains("<Success>");
 		String name = tempFile.getName();
 		TethysReporter.getTethysReporter().addReport(new TethysReport(success, collection, name, name));
-//		error = !success; might be a better options. 
+//		error = !success; might be a better options.
 		if (error) {
 			throw new TethysException("Error posting to Tethys", importReturn);
 		}
 		return success;
 	}
 
+	/**
+	 * Zip an xml file (or any file) into a gz file with a new end
+	 * @param xmlFile
+	 * @return
+	 * @throws FileNotFoundException
+	 * @throws IOException
+	 */
+	private File zipOutputFile(File xmlFile) throws FileNotFoundException, IOException {
+		String zipName = xmlFile.toString() + "-temp-.gz";
+		File zipFile = new File(zipName);
+		GZIPOutputStream opStream = new GZIPOutputStream(new FileOutputStream(zipFile));
+		InputStream fis = new FileInputStream(xmlFile);
+		int chunkSize = 100*1024;
+		byte[] buffer = new byte[chunkSize];
+//		ZipEntry zipEntry = new ZipEntry(xmlFile.getName());
+		int bytesRead;
+		while ((bytesRead = fis.read(buffer)) >= 0) {
+			opStream.write(buffer, 0, bytesRead);
+		}
+
+		opStream.close();
+
+		fis.close();
+
+
+		return zipFile;
+	}
+
 	/**
 	 * Update a document within Tethys. We're assuming that a
 	 * document with the same name in the same collection already
@@ -229,7 +283,7 @@ public class DBXMLConnect {
 	 * the removedocument function
 	 * @param nilusDocument
 	 * @return
-	 * @throws TethysException 
+	 * @throws TethysException
 	 */
 	public boolean updateDocument(Object nilusDocument) throws TethysException {
 		deleteDocument(nilusDocument);
@@ -242,7 +296,7 @@ public class DBXMLConnect {
 	 * class to identify the correct collection.
 	 * @param nilusDocument
 	 * @return
-	 * @throws TethysException 
+	 * @throws TethysException
 	 */
 	public boolean deleteDocument(Object nilusDocument) throws TethysException {
 
@@ -259,7 +313,7 @@ public class DBXMLConnect {
 				<DELETE>
   <ITEM> ['ECoastNARW0'] </ITEM>
 </DELETE>
-An error will throw an exception. 
+An error will throw an exception.
 			 */
 		}
 		catch (Exception e) {
@@ -275,7 +329,7 @@ An error will throw an exception.
 	 * Delete a Deploymnet and any contained Detections document. Doesn't work !
 	 * @param deploymentId
 	 * @return
-	 * @throws TethysException 
+	 * @throws TethysException
 	 */
 	public boolean deleteDeployment(String deploymentId) throws TethysException {
 		ArrayList<String> detDocNames = tethysControl.getDbxmlQueries().getDetectionsDocuments(deploymentId);
@@ -292,8 +346,8 @@ An error will throw an exception.
 	}
 
 	/**
-	 * Remove a document based on a collection name and a cdocument Id. 
-	 * @param collection collection name. 
+	 * Remove a document based on a collection name and a cdocument Id.
+	 * @param collection collection name.
 	 * @param documentName document name (not the internal Document Id)
 	 * @return
 	 * @throws TethysException
@@ -303,8 +357,8 @@ An error will throw an exception.
 	}
 
 	/**
-	 * Remove a document based on a collection name and a document namw. 
-	 * @param collectionName collection name. 
+	 * Remove a document based on a collection name and a document namw.
+	 * @param collectionName collection name.
 	 * @param documentName document name (not the internal Document Id)
 	 * @return
 	 * @throws TethysException
@@ -319,7 +373,7 @@ An error will throw an exception.
 				<DELETE>
   					<ITEM> ['ECoastNARW0'] </ITEM>
 				</DELETE>
-			An error will throw an exception. 
+			An error will throw an exception.
 			 */
 		}
 		catch (Exception e) {
@@ -330,14 +384,14 @@ An error will throw an exception.
 	}
 
 	/**
-	 *  check the return string from importFiles and if it's an 
-	 *  error, throw an exception. Otherwise do nothing.  
+	 *  check the return string from importFiles and if it's an
+	 *  error, throw an exception. Otherwise do nothing.
 	 * @param fileError
 	 */
 	private void checkReturnString(String fileError) {
 		/**
 		 * Example good string is
-		 * 
+		 *
 C:\Users\dg50\AppData\Local\Temp\PAMGuardTethys\20080311_2DSimplex_0.xml: 7360 bytes
 <?xml version="1.0" encoding="iso-8859-1"?>
 <Import>
@@ -367,7 +421,7 @@ C:\Users\dg50\AppData\Local\Temp\PAMGuardTethys\20080311_2DSimplex_0.xmlnot: 0 b
 
 	/**
 	 * Seems we have to get rid of the line <?xml version="1.0" encoding="UTF-8"?>
-	 * which is being put there by the marshaller ? 
+	 * which is being put there by the marshaller ?
 	 * @param tempFile
 	 */
 	private File stripXMLHeader(File tempFile) {
@@ -377,7 +431,7 @@ C:\Users\dg50\AppData\Local\Temp\PAMGuardTethys\20080311_2DSimplex_0.xmlnot: 0 b
 		try {
 			BufferedReader reader = new BufferedReader(new FileReader(tempFile));
 			BufferedWriter writer = new BufferedWriter(new FileWriter(tempTemp));
-			String line = reader.readLine(); 
+			String line = reader.readLine();
 			while (line != null) {
 				// see if the line has any unicode in it
 				int len = line.length();

src/tethys/dbxml/DBXMLQueries.java

@@ -515,6 +515,9 @@ public class DBXMLQueries {
 		 * first query for Detections documents associated with this deployment and datablock.
 		 * updated May 23
 		 */
+		if (dataBlock == null) {
+			return null;
+		}
 		String queryNoDepl = "{\"species\":{\"query\":{\"op\":\"lib:completename2tsn\",\"optype\":\"function\",\"operands\":[\"%s\"]},\"return\":{\"op\":\"lib:tsn2completename\",\"optype\":\"function\",\"operands\":[\"%s\"]}},\"return\":[\"Detections/Id\"],\"select\":[{\"op\":\"=\",\"operands\":[\"Detections/Algorithm/Software\",\"LongDataName\"],\"optype\":\"binary\"}],\"enclose\":1}";
 		String queryWithDepl = "{\"species\":{\"query\":{\"op\":\"lib:completename2tsn\",\"optype\":\"function\",\"operands\":[\"%s\"]},\"return\":{\"op\":\"lib:tsn2completename\",\"optype\":\"function\",\"operands\":[\"%s\"]}},\"return\":[\"Detections/Id\"],\"select\":[{\"op\":\"=\",\"operands\":[\"Detections/DataSource/DeploymentId\",\"TheDeploymentId\"],\"optype\":\"binary\"},{\"op\":\"=\",\"operands\":[\"Detections/Algorithm/Software\",\"LongDataName\"],\"optype\":\"binary\"}],\"enclose\":1}";
 		String query;

src/tethys/deployment/DeploymentHandler.java

@@ -73,6 +73,7 @@ import nilus.UnknownSensor;
 import pamMaths.PamVector;
 import pamMaths.STD;
 import tethys.Collection;
+import tethys.CollectionHandler;
 import tethys.TethysControl;
 import tethys.TethysLocationFuncs;
 import tethys.TethysState;
@@ -83,6 +84,7 @@ import tethys.TethysState.StateType;
 import tethys.dbxml.DBXMLConnect;
 import tethys.dbxml.TethysException;
 import tethys.deployment.swing.DeploymentWizard;
+import tethys.deployment.swing.EffortProblemDialog;
 import tethys.deployment.swing.RecordingGapDialog;
 import tethys.niluswraps.PDeployment;
 import tethys.output.TethysExportParams;
@@ -98,16 +100,9 @@ import tethys.swing.DeploymentTableObserver;
  * @author dg50
  *
  */
-public class DeploymentHandler implements TethysStateObserver, DeploymentTableObserver {
+public class DeploymentHandler extends CollectionHandler implements TethysStateObserver, DeploymentTableObserver {
 	
-	private TethysControl tethysControl;
-	
-	/**
-	 * @return the tethysControl
-	 */
-	public TethysControl getTethysControl() {
-		return tethysControl;
-	}
+//	private TethysControl tethysControl;
 	
 	private EffortFunctions effortFunctions;
 
@@ -119,8 +114,11 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 	
 	private DeploymentExportOpts deploymentExportOptions = new DeploymentExportOpts();
 
+	public static final String helpPoint = "utilities.tethys.docs.deployments";
+
 	public DeploymentHandler(TethysControl tethysControl) {
-		super();
+		
+		super(tethysControl, Collection.Deployments);
 		
 		this.tethysControl = tethysControl;
 		
@@ -368,8 +366,39 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 
 	public void createPamguardOverview() {
 		deploymentOverview = effortFunctions.makeRecordingOverview();
+		
+		checkDeploymentOverview(deploymentOverview);
+		
 		updateProjectDeployments();
 		matchPamguard2Tethys(deploymentOverview, projectDeployments);
+		
+		tethysControl.sendStateUpdate(new TethysState(StateType.NEWPAMGUARDSELECTION));
+	}
+
+	/**
+	 * Check the deployment overview for consistency.<br>
+	 * Take the raw audio information and the binary information and check they are similar.
+	 * if not, ask the user what to do. 
+	 * @param deploymentOverview
+	 */
+	private void checkDeploymentOverview(DeploymentOverview overview) {
+		RecordingList rawList = overview.getRawDataList();
+		RecordingList binList = overview.getBinaryDataList();
+		if (rawList == null || binList == null) {
+			return; // nothing to do
+		}
+		double similarity = rawList.getSimilarity(binList);
+		if (similarity > 0.95) {
+			return;
+		}
+		/*
+		 *  if we get here, it seems like the two lists are very different, so
+		 *  show a dialog to ask the user what to do. 
+		 */
+		RecordingList selList = EffortProblemDialog.showDialog(tethysControl.getGuiFrame(), overview);
+		if (selList != null) {
+			tethysControl.getTethysExportParams().setEffortSourceName(selList.getSourceName());
+		}
 	}
 
 	/**
@@ -381,22 +410,22 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 		if (exportOptions != null) {
 			this.deploymentExportOptions = exportOptions;
 			deploymentOverview = getDeploymentOverview();
-			ArrayList<RecordingPeriod> allPeriods = deploymentOverview.getRecordingPeriods();
+			RecordingList allPeriods = deploymentOverview.getMasterList(getTethysControl());
 			exportDeployments(allPeriods);
 		}
 	}
 
 	/**
 	 * Export deployments docs. Playing with a couple of different ways of doing this. 
-	 * @param selectedDeployments
+	 * @param allPeriods
 	 */
-	public void exportDeployments(ArrayList<RecordingPeriod> selectedDeployments) {
+	public void exportDeployments(RecordingList allPeriods) {
 		TethysReporter.getTethysReporter().clear();
 		if (deploymentExportOptions.separateDeployments) {
-			exportSeparateDeployments(selectedDeployments);
+			exportSeparateDeployments(allPeriods);
 		}
 		else {
-			exportOneDeploymnet(selectedDeployments);
+			exportOneDeploymnet(allPeriods);
 		}
 		TethysReporter.getTethysReporter().showReport(tethysControl.getGuiFrame(), true);
 	}
@@ -404,7 +433,7 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 	/**
 	 * Make one big deployment document with all the recording periods in it. 
 	 */
-	private void exportOneDeploymnet(ArrayList<RecordingPeriod> selectedDeployments) {
+	private void exportOneDeploymnet(RecordingList recordingList) {
 		// do the lot, whatever ...
 		Float sampleRate = null;
 		AcquisitionControl daq = (AcquisitionControl) PamController.getInstance().findControlledUnit(AcquisitionControl.class, null);
@@ -414,10 +443,9 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 			sampleRate = daqParams.sampleRate;
 		}
 		
-		selectedDeployments = getDeploymentOverview().getRecordingPeriods();
 		int freeId = getTethysControl().getDeploymentHandler().getFirstFreeDeploymentId();
-		RecordingPeriod onePeriod = new RecordingPeriod(selectedDeployments.get(0).getRecordStart(), 
-				selectedDeployments.get(selectedDeployments.size()-1).getRecordStop());
+		RecordingPeriod onePeriod = new RecordingPeriod(recordingList.getStart(), 
+				recordingList.getEnd());
 		TethysExportParams exportParams = tethysControl.getTethysExportParams();
 		String id = String.format("%s_%s", exportParams.getDatasetName(), "all");
 		Deployment deployment = createDeploymentDocument(freeId, onePeriod, id);
@@ -425,7 +453,8 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 		Deployment globalMeta = getTethysControl().getGlobalDeplopymentData();
 		deployment.setCruise(globalMeta.getCruise());
 		deployment.setSite(globalMeta.getSite());
-		if (selectedDeployments.size() > 1) {
+		ArrayList<RecordingPeriod> effortPeriods = recordingList.getEffortPeriods();
+		if (recordingList.size() > 1) {
 //			// now need to remove the sampling details - don't though, add invalid periods instead. 
 //			SamplingDetails samplingDetails = deployment.getSamplingDetails();
 //			samplingDetails.getChannel().clear();
@@ -440,9 +469,9 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 				deployment.setQualityAssurance(qa = new AcousticDataQAType());
 			}
 			List<Quality> qualityList = qa.getQuality();
-			for (int i = 1; i < selectedDeployments.size(); i++) {
-				long end = selectedDeployments.get(i-1).getRecordStop();
-				long start = selectedDeployments.get(i).getRecordStart();
+			for (int i = 1; i < recordingList.size(); i++) {
+				long end = effortPeriods.get(i-1).getRecordStop();
+				long start = effortPeriods.get(i).getRecordStart();
 				Quality q = new Quality();
 				q.setStart(TethysTimeFuncs.xmlGregCalFromMillis(end));
 				q.setEnd(TethysTimeFuncs.xmlGregCalFromMillis(start));
@@ -479,14 +508,15 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 	/**
 	 * Make a separate deployment document for every recording period. 
 	 */
-	private void exportSeparateDeployments(ArrayList<RecordingPeriod> selectedDeployments) {
+	private void exportSeparateDeployments(RecordingList recordingList) {
 		
 		int freeId = getTethysControl().getDeploymentHandler().getFirstFreeDeploymentId();
 		// fill in a few things from here
 		Deployment globalMeta = getTethysControl().getGlobalDeplopymentData();
 		TethysExportParams exportParams = tethysControl.getTethysExportParams();
-		for (int i = 0; i < selectedDeployments.size(); i++) {
-			RecordingPeriod recordPeriod = selectedDeployments.get(i);
+		ArrayList<RecordingPeriod> effortPeriods = recordingList.getEffortPeriods();
+		for (int i = 0; i < recordingList.size(); i++) {
+			RecordingPeriod recordPeriod = effortPeriods.get(i);
 			PDeployment exDeploymnet = recordPeriod.getMatchedTethysDeployment();
 			Deployment deployment = null;
 			String id = String.format("%s_%d", exportParams.getDatasetName(), i);
@@ -532,8 +562,9 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 		if (deployments == null || deploymentOverview == null) {
 			return;
 		}
-		ArrayList<RecordingPeriod> recordingPeriods = deploymentOverview.getRecordingPeriods();
-		for (RecordingPeriod aPeriod : recordingPeriods) {
+		RecordingList recordingList = deploymentOverview.getMasterList(getTethysControl());
+		ArrayList<RecordingPeriod> effortPeriods = recordingList.getEffortPeriods();
+		for (RecordingPeriod aPeriod : effortPeriods) {
 			PDeployment closestDeployment = findClosestDeployment(aPeriod, deployments);
 			aPeriod.setMatchedTethysDeployment(closestDeployment);
 			if (closestDeployment != null) {
@@ -592,7 +623,8 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 		if (deploymentOverview == null) {
 			return matched;
 		}
-		for (RecordingPeriod period : deploymentOverview.getRecordingPeriods()) {
+		ArrayList<RecordingPeriod> effortPeriods = deploymentOverview.getMasterList(getTethysControl()).getEffortPeriods();
+		for (RecordingPeriod period : effortPeriods) {
 			PDeployment deployment = period.getMatchedTethysDeployment();
 			if (deployment != null) {
 				if (matched.contains(deployment) == false) {
@@ -1230,7 +1262,8 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 			}
 			regimens.add(regimen);
 			
-			DutyCycleInfo dutyCycleInf = deploymentOverview.getDutyCycleInfo();
+			RecordingList recordingList = deploymentOverview.getMasterList(getTethysControl());
+			DutyCycleInfo dutyCycleInf = recordingList.assessDutyCycle();
 			boolean isDS = dutyCycleInf != null && dutyCycleInf.isDutyCycled;
 			if (isDS) {
 				DutyCycle dutyCycle = new DutyCycle();
@@ -1307,4 +1340,9 @@ public class DeploymentHandler implements TethysStateObserver, DeploymentTableOb
 		return deploymentExportOptions;
 	}
 
+	@Override
+	public String getHelpPoint() {
+		return helpPoint;
+	}
+
 }

src/tethys/deployment/DeploymentOverview.java

@@ -1,16 +1,6 @@
 package tethys.deployment;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.ListIterator;
-
-import Acquisition.AcquisitionControl;
-import Acquisition.AcquisitionParameters;
-import Acquisition.DaqStatusDataUnit;
-import PamController.PamControlledUnit;
-import PamController.PamController;
-import PamguardMVC.PamDataBlock;
+import tethys.TethysControl;
 
 /**
  * Class to give a general overview of all the effort in PAMGuard which will form the
@@ -22,59 +12,83 @@ import PamguardMVC.PamDataBlock;
  */
 public class DeploymentOverview {
 	
-	private ArrayList<RecordingPeriod> recordingPeriods = new ArrayList<>();
+	private RecordingList rawDataList;
 	
-	private DutyCycleInfo dutyCycleInfo;
+	private RecordingList binaryDataList;
+	
+//	private DutyCycleInfo dutyCycleInfo;
 
-	public DeploymentOverview(DutyCycleInfo dutyCycleInfo) {
-		super();
-		this.dutyCycleInfo = dutyCycleInfo;
-	}
+//	public DeploymentOverview(DutyCycleInfo dutyCycleInfo) {
+//		super();
+//		this.dutyCycleInfo = dutyCycleInfo;
+//	}
 	
-	public DeploymentOverview(DutyCycleInfo dutyCycleInfo, ArrayList<RecordingPeriod> tempPeriods) {
-		this.dutyCycleInfo = dutyCycleInfo;
-		this.recordingPeriods = tempPeriods;
-	}
-	
-	
-	public void addRecordingPeriod(long start, long stop) {
-		addRecordingPeriod(new RecordingPeriod(start, stop));
+	public DeploymentOverview(DutyCycleInfo dutyCycleInfo, RecordingList rawDataList, RecordingList binaryDataList) {
+//		this.dutyCycleInfo = dutyCycleInfo;
+		this.rawDataList = rawDataList;
+		this.binaryDataList = binaryDataList;
 	}
 
-	private void addRecordingPeriod(RecordingPeriod recordingPeriod) {
-		recordingPeriods.add(recordingPeriod);
+	/**
+	 * @return the rawDataList
+	 */
+	public RecordingList getRawDataList() {
+		return rawDataList;
 	}
 
-	public ArrayList<RecordingPeriod> getRecordingPeriods() {
-		return recordingPeriods;
+	/**
+	 * @return the binaryDataList
+	 */
+	public RecordingList getBinaryDataList() {
+		return binaryDataList;
 	}
 
-	public DutyCycleInfo getDutyCycleInfo() {
-		return dutyCycleInfo;
+//	/**
+//	 * @return the dutyCycleInfo
+//	 */
+//	public DutyCycleInfo getDutyCycleInfo() {
+//		return dutyCycleInfo;
+//	}
+	
+	public RecordingList getMasterList(TethysControl tethysControl) {
+		return getMasterList(tethysControl.getTethysExportParams().getEffortSourceName());
+	}
+
+	public RecordingList getMasterList(String effortSourceName) {
+		if (effortSourceName == null) {
+			return getLongestList();
+		}
+		if (binaryDataList != null & binaryDataList.getSourceName().equals(effortSourceName)) {
+			return binaryDataList;
+		}
+		if (rawDataList != null & rawDataList.getSourceName().equals(effortSourceName)) {
+			return rawDataList;
+		}
+		return getLongestList();
 	}
 	
 	/**
-	 * Get the start time of the first recording
+	 * Get the recording list with the greatest duration (start to end)
+	 * not looking at coverage between those times. 
 	 * @return
 	 */
-	public Long getFirstStart() {
-		if (recordingPeriods.size() > 0) {
-			return recordingPeriods.get(0).getRecordStart();
+	public RecordingList getLongestList() {
+		if (binaryDataList == null) {
+			return rawDataList;
 		}
-		return null;
-	}
-	
-	/**
-	 * Get the end time of the last recording
-	 * @return
-	 */
-	public Long getLastEnd() {
-		if (recordingPeriods.size() > 0) {
-			return recordingPeriods.get(recordingPeriods.size()-1).getRecordStop();
+		if (rawDataList == null) {
+			return binaryDataList;
+		}
+		long lRaw = rawDataList.duration();
+		long lBin = binaryDataList.duration();
+		
+		if (lRaw > lBin) {
+			return rawDataList;
+		}
+		else {
+			return binaryDataList;
 		}
-		return null;
 	}
-	
-	
+
 
 }

src/tethys/deployment/EffortFunctions.java

@@ -41,36 +41,40 @@ public class EffortFunctions {
 		this.tethysControl = tethysControl;
 	}
 
-	private DeploymentOverview createOverview(RecordingList tempPeriods) {
-
-		DutyCycleInfo dutyCycleinfo = assessDutyCycle(tempPeriods);
-		if (dutyCycleinfo == null) {
-			return null;
-		}
-
-		// if it's duty cycles, then we only want a single entry. 
-		RecordingList deploymentPeriods;
-		if (dutyCycleinfo.isDutyCycled == false) {
-			deploymentPeriods = tempPeriods;
-		}
-		else {
-			deploymentPeriods = new RecordingList();
-			deploymentPeriods.add(new RecordingPeriod(tempPeriods.get(0).getRecordStart(), tempPeriods.get(tempPeriods.size()-1).getRecordStop()));
-		}
-		/*
-		 * do another sort of the deploymentPeriods. The start stops were in the order they went into the 
-		 * database in the hope that pairs were the right way round. Now check all data are/
-		 */
-		Collections.sort(deploymentPeriods, new Comparator<RecordingPeriod>() {
-			@Override
-			public int compare(RecordingPeriod o1, RecordingPeriod o2) {
-				return (int) (o1.getRecordStart()-o2.getRecordStart());
-			}
-		});
-
-		DeploymentOverview deploymentOverview = new DeploymentOverview(dutyCycleinfo, deploymentPeriods);
-		return deploymentOverview;
-	}
+//	private DeploymentOverview createOverview(RecordingList tempPeriods) {
+//
+//		tempPeriods.sort();
+//		
+//		DutyCycleInfo dutyCycleinfo = assessDutyCycle(tempPeriods);
+//		if (dutyCycleinfo == null) {
+//			return null;
+//		}
+//		
+//
+//		// if it's duty cycles, then we only want a single entry. 
+//		RecordingList deploymentPeriods;
+//		if (dutyCycleinfo.isDutyCycled == false) {
+//			deploymentPeriods = tempPeriods;
+//		}
+//		else {
+//			deploymentPeriods = new RecordingList(tempPeriods.getSourceName());
+//			deploymentPeriods.add(new RecordingPeriod(tempPeriods.get(0).getRecordStart(), tempPeriods.get(tempPeriods.size()-1).getRecordStop()));
+//		}
+//		/*
+//		 * do another sort of the deploymentPeriods. The start stops were in the order they went into the 
+//		 * database in the hope that pairs were the right way round. Now check all data are/
+//		 */
+//		deploymentPeriods.sort();
+////		Collections.sort(deploymentPeriods, new Comparator<RecordingPeriod>() {
+////			@Override
+////			public int compare(RecordingPeriod o1, RecordingPeriod o2) {
+////				return (int) (o1.getRecordStart()-o2.getRecordStart());
+////			}
+////		});
+//
+//		DeploymentOverview deploymentOverview = new DeploymentOverview(dutyCycleinfo, deploymentPeriods);
+//		return deploymentOverview;
+//	}
 
 
 	public DeploymentOverview makeRecordingOverview() {
@@ -79,13 +83,18 @@ public class EffortFunctions {
 		
 		RecordingList binaryPeriods = listBinaryFiles();
 		
-		long l1 = listDuration(recordingPeriods);
-		long l2 = listDuration(binaryPeriods);
-		if (listDuration(binaryPeriods) > listDuration(recordingPeriods)) {
-			recordingPeriods = binaryPeriods;
-		}
+		// see what the similarity is between them
+//		double sim = recordingPeriods.getSimilarity(binaryPeriods);
+//		double testSim = recordingPeriods.getSimilarity(recordingPeriods);
 		
-		DeploymentOverview deploymentOverview = createOverview(recordingPeriods);
+//		long l1 = listDuration(recordingPeriods);
+//		long l2 = listDuration(binaryPeriods);
+//		if (listDuration(binaryPeriods) > listDuration(recordingPeriods)) {
+//			recordingPeriods = binaryPeriods;
+//		}
+//		
+//		DeploymentOverview deploymentOverview = createOverview(recordingPeriods);
+		DeploymentOverview deploymentOverview = new DeploymentOverview(null, recordingPeriods, binaryPeriods);
 		
 		return deploymentOverview;
 	}
@@ -128,7 +137,8 @@ public class EffortFunctions {
 				}
 			}
 		}
-		bestList = mergeRecordings(bestList);
+		DeploymentExportOpts exportOptions = tethysControl.getDeploymentHandler().getDeploymentExportOptions();
+		bestList.mergeRecordingPeriods(exportOptions.maxRecordingGapSeconds*1000);
 		return bestList;
 	}
 	
@@ -138,7 +148,7 @@ public class EffortFunctions {
 		if (mapPoints == null) {
 			return null;
 		}
-		RecordingList periods = new RecordingList();
+		RecordingList periods = new RecordingList(dataMap.getDataMapName());
 		for (OfflineDataMapPoint mapPoint : mapPoints) {
 			periods.add(new RecordingPeriod(mapPoint.getStartTime(), mapPoint.getEndTime()));
 		}
@@ -219,117 +229,60 @@ public class EffortFunctions {
 		//							PamCalendar.formatDBDateTime(aP.getRecordStop()));
 		//				}
 
-		tempPeriods = mergeRecordings(tempPeriods);
-
-		return tempPeriods;
-	}
-
-	/**
-	 * Merge close recordings and discard ones that are too short. 
-	 * @param tempPeriods all recording periods, may be from consecutive files. 
-	 * @return merged list. 
-	 */
-	private RecordingList mergeRecordings(RecordingList tempPeriods) {
-		// now go through those and merge into longer periods where there is no gap between files.
-		if (tempPeriods == null) {
-			return null;
-		}
-
+		tempPeriods.sort();
 		DeploymentExportOpts exportOptions = tethysControl.getDeploymentHandler().getDeploymentExportOptions();
-
-		ListIterator<RecordingPeriod> iterator = tempPeriods.listIterator();
-		RecordingPeriod prevPeriod = null;
-		while (iterator.hasNext()) {
-			RecordingPeriod nextPeriod = iterator.next();
-			long nextDur = nextPeriod.getRecordStop()-nextPeriod.getRecordStart();
-			if (nextDur == 0) {
-				continue;
-			}
-			if (prevPeriod != null) {
-				long gap = nextPeriod.getRecordStart() - prevPeriod.getRecordStop();
-				long prevDur = prevPeriod.getRecordStop()-prevPeriod.getRecordStart();
-				if (gap < exportOptions.maxRecordingGapSeconds*1000) {
-					// ignoring up to 3s gap or a sample error < 2%.Dunno if this is sensible or not.
-					prevPeriod.setRecordStop(nextPeriod.getRecordStop());
-					iterator.remove();
-					nextPeriod = prevPeriod;
-				}
-			}
-			prevPeriod = nextPeriod;
-		}
-		// now remove ones which are too short even after merging. 
-		iterator = tempPeriods.listIterator();
-		while (iterator.hasNext()) {
-			RecordingPeriod nextPeriod = iterator.next();
-			long duration = nextPeriod.getDuration();
-			if (duration < exportOptions.minRecordingLengthSeconds*1000L) {
-				iterator.remove();
-			}
-		}
+		tempPeriods.mergeRecordingPeriods(exportOptions.maxRecordingGapSeconds*1000);
 
 		return tempPeriods;
 	}
 
-	/**
-	 * Work out whether or not the data are evenly duty cycled by testing the
-	 * distributions of on and off times.
-	 * @param tempPeriods
-	 * @return
-	 */
-	private DutyCycleInfo assessDutyCycle(RecordingList tempPeriods) {
-		if (tempPeriods == null) {
-			return null;
-		}
-		int n = tempPeriods.size();
-		if (n < 2) {
-			return new DutyCycleInfo(false, 0,0,n);
-		}
-		double[] ons = new double[n-1]; // ignore the last one since it may be artificially shortened which is OK
-		double[] gaps = new double[n-1];
-		for (int i = 0; i < n-1; i++) {
-			ons[i] = tempPeriods.get(i).getDuration()/1000.;
-			gaps[i] = (tempPeriods.get(i+1).getRecordStart()-tempPeriods.get(i).getRecordStop())/1000.;
-		}
-		/* now look at how consistent those values are
-		 * But some data gets messed by small gaps, so want to 
-		 * remove outliers and concentrate on say 80% of the data. 
-		 */
-		ons = getDistributionCentre(ons, 80);
-		gaps = getDistributionCentre(gaps, 80);
-		Arrays.sort(gaps);
+//	/**
+//	 * Merge close recordings and discard ones that are too short. 
+//	 * @param tempPeriods all recording periods, may be from consecutive files. 
+//	 * @return merged list. 
+//	 */
+//	private RecordingList mergeRecordings(RecordingList tempPeriods) {
+//		// now go through those and merge into longer periods where there is no gap between files.
+//		if (tempPeriods == null) {
+//			return null;
+//		}
+//
+//		DeploymentExportOpts exportOptions = tethysControl.getDeploymentHandler().getDeploymentExportOptions();
+//
+//		ListIterator<RecordingPeriod> iterator = tempPeriods.listIterator();
+//		RecordingPeriod prevPeriod = null;
+//		while (iterator.hasNext()) {
+//			RecordingPeriod nextPeriod = iterator.next();
+//			long nextDur = nextPeriod.getRecordStop()-nextPeriod.getRecordStart();
+//			if (nextDur == 0) {
+//				continue;
+//			}
+//			if (prevPeriod != null) {
+//				long gap = nextPeriod.getRecordStart() - prevPeriod.getRecordStop();
+//				long prevDur = prevPeriod.getRecordStop()-prevPeriod.getRecordStart();
+//				if (gap < exportOptions.maxRecordingGapSeconds*1000) {
+//					// ignoring up to 3s gap or a sample error < 2%.Dunno if this is sensible or not.
+//					prevPeriod.setRecordStop(nextPeriod.getRecordStop());
+//					iterator.remove();
+//					nextPeriod = prevPeriod;
+//				}
+//			}
+//			prevPeriod = nextPeriod;
+//		}
+//		// now remove ones which are too short even after merging. 
+//		iterator = tempPeriods.listIterator();
+//		while (iterator.hasNext()) {
+//			RecordingPeriod nextPeriod = iterator.next();
+//			long duration = nextPeriod.getDuration();
+//			if (duration < exportOptions.minRecordingLengthSeconds*1000L) {
+//				iterator.remove();
+//			}
+//		}
+//
+//		return tempPeriods;
+//	}
 
 
-		STD std = new STD();
-		double onsMean = std.getMean(ons);
-		double onsSTD = std.getSTD(ons);
-		double gapsMean = std.getMean(gaps);
-		double gapsSTD = std.getSTD(gaps);
-		boolean dutyCycle = onsSTD/onsMean < .05 && gapsSTD/gapsMean < 0.05;
-		DutyCycleInfo cycleInfo = new DutyCycleInfo(dutyCycle, onsMean, gapsMean, tempPeriods.size());
-		return cycleInfo;
-	}
-
-	/**
-	 * Get the central part of a distribution without any outliers so 
-	 * that we can get a better assessment of duty cycle. 
-	 * @param data unsorted distribution data. 
-	 * @param percent percentage to include (half this removed from top and bottom)
-	 * @return
-	 */
-	private double[] getDistributionCentre(double[] data, double percent) {
-		if (data == null) {
-			return null;
-		}
-		Arrays.sort(data);
-		int nRem = (int) Math.round(data.length * (100-percent)/200);
-		int newLen = data.length-nRem*2;
-		double[] subdata = Arrays.copyOfRange(data, nRem, data.length-2*nRem);
-		if (subdata.length < 2) {
-			return data;
-		}
-		return subdata;
-	}
-
 
 	/**
 	 * Get data times from any other datamap, since this will generally match the acquisition anyway
@@ -360,16 +313,17 @@ public class EffortFunctions {
 			return null;
 		}
 		// get the times out of it. 
-		RecordingList recPeriods = new RecordingList();
+		RecordingList recPeriods = new RecordingList(bestMap.getDataMapName());
 		List<OfflineDataMapPoint> mapPoints = bestMap.getMapPoints();
 		for (OfflineDataMapPoint mapPoint : mapPoints) {
 			recPeriods.add(new RecordingPeriod(mapPoint.getStartTime(), mapPoint.getEndTime()));
+			recPeriods.add(mapPoint.getStartTime(), mapPoint.getEndTime());
 		}
 		return recPeriods;
 	}
 
 	private RecordingList extractTimesFromStatus(ArrayList<DaqStatusDataUnit> allStatusData) {
-		RecordingList tempPeriods = new RecordingList();
+		RecordingList tempPeriods = new RecordingList("Data acquisition status");
 		long dataStart = Long.MAX_VALUE;
 		long dataEnd = Long.MIN_VALUE;
 		Long lastStart = null;

src/tethys/deployment/RecordingList.java

@@ -1,12 +1,43 @@
 package tethys.deployment;
 
+import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.Iterator;
 
-public class RecordingList extends ArrayList<RecordingPeriod> {
+import PamUtils.PamCalendar;
+import pamMaths.STD;
+
+/**
+ * Information about periods of effort that might come from either the raw data recordings or 
+ * an analysis of binary data maps. 
+ * @author dg50
+ *
+ */
+public class RecordingList implements Serializable {
 
 	private static final long serialVersionUID = 1L;
+	
+	private ArrayList<RecordingPeriod> effortPeriods = new ArrayList();
+	
+	/**
+	 * Name / source of this list. 
+	 */
+	private String sourceName;
+
+	/**
+	 * @param sourceName
+	 */
+	public RecordingList(String sourceName) {
+		this.sourceName = sourceName;
+	}
+
+	public RecordingList(String sourceName, ArrayList<RecordingPeriod> selectedDeployments) {
+		this.sourceName = sourceName;
+		this.effortPeriods = selectedDeployments;
+	}
 
 	/**
 	 * Get the duration of the recording periods from start to end. 
@@ -21,27 +52,27 @@ public class RecordingList extends ArrayList<RecordingPeriod> {
 	 * @return
 	 */
 	public long getStart() {
-		if (size() == 0) {
+		if (effortPeriods.size() == 0) {
 			return 0;
 		}
-		return get(0).getRecordStart();
+		return effortPeriods.get(0).getRecordStart();
 	}
 	
 	/**
 	 * get the end of the last in the list. 
 	 */
 	public long getEnd() {
-		if (size() == 0) {
+		if (effortPeriods.size() == 0) {
 			return 0;
 		}
-		return get(size()-1).getRecordStop();
+		return effortPeriods.get(effortPeriods.size()-1).getRecordStop();
 	}
 	
 	/**
 	 * Sort the list in ascending order. 
 	 */
 	public void sort() {
-		Collections.sort(this, new Comparator<RecordingPeriod>() {
+		Collections.sort(effortPeriods, new Comparator<RecordingPeriod>() {
 
 			@Override
 			public int compare(RecordingPeriod o1, RecordingPeriod o2) {
@@ -49,4 +80,172 @@ public class RecordingList extends ArrayList<RecordingPeriod> {
 			}
 		});
 	}
+	
+	/**
+	 * Get the coverage as a fraction. This is the sum of the individual periods divided 
+	 * by the start to end times
+	 * @return
+	 */
+	public double getCoverage() {
+		long cov = 0;
+		long durTot = 0;
+		if (effortPeriods.size() == 0) {
+			return 0;
+		}
+		Iterator<RecordingPeriod> it = effortPeriods.iterator();
+		while (it.hasNext()) {
+			RecordingPeriod rp = it.next();
+			cov += rp.getDuration();
+		}
+		durTot = getEnd()-getStart();
+		return (double) cov / (double) durTot;
+	}
+	
+	/**
+	 * Merge recording periods, with a max gap between periods in milliseconds. 
+	 * @param maxGap
+	 * @return the number of periods removed. 
+	 */
+	public int mergeRecordingPeriods(long maxGap) {
+		if (effortPeriods.size() < 2) {
+			return 0;
+		}
+		Iterator<RecordingPeriod> it = effortPeriods.iterator();
+		RecordingPeriod prev = it.next();
+		int removed = 0;
+		while (it.hasNext()) {
+			RecordingPeriod curr = it.next();
+			if (curr.getRecordStart() - prev.getRecordStop() <= maxGap) {
+				prev.setRecordStop(curr.getRecordStop());
+				it.remove();
+				removed++;
+			}
+			else {
+				prev = curr;
+			}
+		}
+		return removed;
+	}
+	
+	/**
+	 * Work out whether or not the data are evenly duty cycled by testing the
+	 * distributions of on and off times.
+	 * @param tempPeriods
+	 * @return
+	 */
+	public DutyCycleInfo assessDutyCycle() {
+		if (effortPeriods == null) {
+			return null;
+		}
+		int n = effortPeriods.size();
+		if (n < 2) {
+			return new DutyCycleInfo(false, 0,0,n);
+		}
+		double[] ons = new double[n-1]; // ignore the last one since it may be artificially shortened which is OK
+		double[] gaps = new double[n-1];
+		for (int i = 0; i < n-1; i++) {
+			ons[i] = effortPeriods.get(i).getDuration()/1000.;
+			gaps[i] = (effortPeriods.get(i+1).getRecordStart()-effortPeriods.get(i).getRecordStop())/1000.;
+		}
+		/* now look at how consistent those values are
+		 * But some data gets messed by small gaps, so want to 
+		 * remove outliers and concentrate on say 80% of the data. 
+		 */
+		ons = getDistributionCentre(ons, 80);
+		gaps = getDistributionCentre(gaps, 80);
+		Arrays.sort(gaps);
+
+		STD std = new STD();
+		double onsMean = std.getMean(ons);
+		double onsSTD = std.getSTD(ons);
+		double gapsMean = std.getMean(gaps);
+		double gapsSTD = std.getSTD(gaps);
+		boolean dutyCycle = onsSTD/onsMean < .05 && gapsSTD/gapsMean < 0.05;
+		DutyCycleInfo cycleInfo = new DutyCycleInfo(dutyCycle, onsMean, gapsMean, effortPeriods.size());
+		return cycleInfo;
+	}
+	/**
+	 * Get the central part of a distribution without any outliers so 
+	 * that we can get a better assessment of duty cycle. 
+	 * @param data unsorted distribution data. 
+	 * @param percent percentage to include (half this removed from top and bottom)
+	 * @return
+	 */
+	private double[] getDistributionCentre(double[] data, double percent) {
+		if (data == null) {
+			return null;
+		}
+		Arrays.sort(data);
+		int nRem = (int) Math.round(data.length * (100-percent)/200);
+		int newLen = data.length-nRem*2;
+		double[] subdata = Arrays.copyOfRange(data, nRem, data.length-2*nRem);
+		if (subdata.length < 2) {
+			return data;
+		}
+		return subdata;
+	}
+
+	/**
+	 * @return the sourceName
+	 */
+	public String getSourceName() {
+		return sourceName;
+	}
+
+	@Override
+	public String toString() {
+		if (effortPeriods.size() == 0) {
+			return "Empty recording list";
+		}
+		String str = String.format("%s: %s to %s, %3.1f%% coverage", getSourceName(),
+				PamCalendar.formatDBDateTime(getStart()), 
+				PamCalendar.formatDBDateTime(getEnd()), getCoverage()*100);
+		return str;
+	}
+	
+	/**
+	 * Get similarity to another recording list. 1 = identical, 0 means not even overlapping.
+	 * @param other other recording list. 
+	 * @return measure of similarity. 
+	 */
+	public double getSimilarity(RecordingList other) {
+		double sim1 = (double) other.duration() / (double) this.duration();
+		if (sim1 > 1) {
+			sim1 = 1./sim1;
+		}
+		long overlap = Math.min(other.getEnd(), this.getEnd()) - Math.max(other.getStart(), this.getStart());
+		overlap = Math.max(0,  overlap);
+		long longest = Math.max(other.duration(), this.duration());
+		double sim2 = (double) overlap / (double) longest;
+		
+		return Math.min(sim1, sim2);
+	}
+
+	/**
+	 * Add a recording period to the list. 
+	 * @param recordingPeriod
+	 */
+	public void add(RecordingPeriod recordingPeriod) {
+		effortPeriods.add(recordingPeriod);
+	}
+
+	/**
+	 * Add a recording period to the list. 
+	 * @param startTime
+	 * @param endTime
+	 */
+	public void add(long startTime, long endTime) {
+		add (new RecordingPeriod(startTime, endTime));
+	}
+
+	public int size() {
+		return effortPeriods.size();
+	}
+
+	/**
+	 * @return the effortPeriods
+	 */
+	public ArrayList<RecordingPeriod> getEffortPeriods() {
+		return effortPeriods;
+	}
 }

src/tethys/deployment/RecordingPeriod.java

@@ -1,5 +1,6 @@
 package tethys.deployment;
 
+import PamUtils.PamCalendar;
 import tethys.niluswraps.PDeployment;
 
 public class RecordingPeriod {
@@ -71,5 +72,11 @@ public class RecordingPeriod {
 		selected = !selected;
 		return selected;
 	}
+
+	@Override
+	public String toString() {
+		return String.format("%s to %s, %s", PamCalendar.formatDBDateTime(recordStart), 
+				PamCalendar.formatDBDateTime(recordStop), PamCalendar.formatDuration(getDuration()));
+	}
 	
 }

src/tethys/deployment/swing/EffortProblemDialog.java

@@ -0,0 +1,163 @@
+package tethys.deployment.swing;
+
+import java.awt.BorderLayout;
+import java.awt.GridBagConstraints;
+import java.awt.GridBagLayout;
+import java.awt.GridLayout;
+import java.awt.Window;
+
+import javax.swing.BoxLayout;
+import javax.swing.ButtonGroup;
+import javax.swing.JCheckBox;
+import javax.swing.JLabel;
+import javax.swing.JPanel;
+import javax.swing.JRadioButton;
+import javax.swing.border.TitledBorder;
+
+import PamUtils.PamCalendar;
+import PamView.dialog.PamDialog;
+import PamView.dialog.PamGridBagContraints;
+import tethys.deployment.DeploymentOverview;
+import tethys.deployment.RecordingList;
+
+/**
+ * Handle problems when binary and raw effort don't add up
+ * @author dg50
+ *
+ */
+public class EffortProblemDialog extends PamDialog {
+	
+	private JRadioButton useRaw, useBinary, useNeither;
+	
+	private JLabel generalInfo;
+	
+	private InfoSet[] infoSets = new InfoSet[2];
+
+	private RecordingList chosenList;
+
+	private DeploymentOverview deploymentOverview;
+	
+	private static EffortProblemDialog singleInstance;
+	
+	private static final String[] setNames = {"Raw data", "Binary data"};
+
+	private EffortProblemDialog(Window parentFrame) {
+		super(parentFrame, "Deployment Effort", false);
+		JPanel mainPanel = new JPanel();
+		mainPanel.setLayout(new BorderLayout());
+		mainPanel.setBorder(new TitledBorder("Effort information"));
+		String info = "<html>There is a mismatch between the time period covered by the raw<br>"
+				+ "data recordings and the time covered in the binary data.<br> "
+				+ "Select the one you wish to use, or Cancel and sort out your data<br>"
+				+ "prior to restarting the Tethys export process</html>";
+		generalInfo = new JLabel(info);
+//		generalInfo.setBorder(new TitledBorder("General"));
+		mainPanel.add(generalInfo, BorderLayout.NORTH);
+		JPanel botPanel = new JPanel(new GridLayout(2, 1));
+		mainPanel.add(botPanel, BorderLayout.CENTER);
+		ButtonGroup bg = new ButtonGroup();
+		for (int i = 0; i < 2; i++) {
+			GridBagConstraints c = new PamGridBagContraints();
+			JPanel subPanel = new JPanel(new GridBagLayout());
+			botPanel.add(subPanel);
+			infoSets[i] = new InfoSet(setNames[i]);
+			c.gridwidth = 2;
+			subPanel.add(infoSets[i].name, c);
+			c.gridx += c.gridwidth;
+			subPanel.add(infoSets[i].select, c);
+			c.gridx = 0;
+			c.gridy++;
+			c.gridwidth = 1;
+			subPanel.add(new JLabel("Start: ", JLabel.RIGHT), c);
+			c.gridx++;
+			subPanel.add(infoSets[i].start, c);
+			c.gridx++;
+			subPanel.add(new JLabel("End: ", JLabel.RIGHT), c);
+			c.gridx++;
+			subPanel.add(infoSets[i].end, c);
+			c.gridy++;
+			c.gridx = 0;
+			subPanel.add(new JLabel("Duration: ", JLabel.RIGHT), c);
+			c.gridx++;
+			subPanel.add(infoSets[i].duration, c);
+			c.gridx++;
+			subPanel.add(new JLabel("Coverage: ", JLabel.RIGHT), c);
+			c.gridx++;
+			subPanel.add(infoSets[i].occupancy, c);
+
+			bg.add(infoSets[i].select);
+		}
+		
+		setDialogComponent(mainPanel);
+		setResizable(true);
+	}
+	
+	public static RecordingList showDialog(Window parentFrame, DeploymentOverview deploymentOverview) {
+		singleInstance = new EffortProblemDialog(parentFrame);
+		singleInstance.setData(deploymentOverview);
+		singleInstance.setVisible(true);
+		return singleInstance.chosenList;
+	}
+
+	private void setData(DeploymentOverview deploymentOverview) {
+		this.deploymentOverview = deploymentOverview;
+		RecordingList rl;
+		for (int i = 0; i < 2; i++) {
+			if (i == 0) {
+				rl = deploymentOverview.getRawDataList();
+			}
+			else {
+				rl = deploymentOverview.getBinaryDataList();
+			}
+			infoSets[i].start.setText(PamCalendar.formatDBDateTime(rl.getStart()));
+			infoSets[i].end.setText(PamCalendar.formatDBDateTime(rl.getEnd()));
+			infoSets[i].duration.setText(PamCalendar.formatDuration(rl.duration()));
+			infoSets[i].occupancy.setText(String.format("%3.0f%%", rl.getCoverage()*100.));
+		}
+		invalidate();
+		pack();
+	}
+
+	@Override
+	public boolean getParams() {
+		if (infoSets[0].select.isSelected()) {
+			chosenList = deploymentOverview.getRawDataList();
+			return true;
+		}
+		if (infoSets[1].select.isSelected()) {
+			chosenList = deploymentOverview.getBinaryDataList();
+			return true;
+		}
+		return false;
+	}
+
+	@Override
+	public void cancelButtonPressed() {
+		// TODO Auto-generated method stub
+
+	}
+
+	@Override
+	public void restoreDefaultSettings() {
+		// TODO Auto-generated method stub
+
+	}
+	
+	private class InfoSet {
+		JLabel name, start, end, duration, occupancy;
+		JCheckBox select;
+		/**
+		 * 
+		 */
+		public InfoSet(String name) {
+			super();
+			this.name = new JLabel(name);
+			this.start = new JLabel("                   ");
+			this.end = new JLabel("                        ");
+			this.select = new JCheckBox("Select " + name);
+			duration = new JLabel(" ");
+			occupancy = new JLabel(" ");
+		}
+	}
+
+}

src/tethys/deployment/swing/ProjectInformationPanel.java

@@ -23,6 +23,7 @@ import nilus.Deployment;
 import tethys.TethysControl;
 import tethys.swing.NewProjectDialog;
 import tethys.swing.SelectProjectDialog;
+import tethys.tooltips.TethysTips;
 
 /**
  * Panel for entering project information
@@ -109,6 +110,12 @@ public class ProjectInformationPanel {
 			});
 		}
 		
+		project.setToolTipText(TethysTips.findTip(Deployment.class, "Project"));
+		cruise.setToolTipText(TethysTips.findTip(Deployment.class, "Cruise"));
+		region.setToolTipText(TethysTips.findTip(Deployment.class, "Region"));
+		site.setToolTipText(TethysTips.findTip(Deployment.class, "Site"));
+		
+		
 	}
 
 	/**

src/tethys/deployment/swing/RecordingGapDialog.java

@@ -36,7 +36,7 @@ public class RecordingGapDialog extends PamDialog {
 		c.gridx++;
 		mainPanel.add(new JLabel(" seconds", JLabel.RIGHT), c);
 		
-		maxGap.setToolTipText("Maximum gap between recording periods. Periods with a gap less than this will be counted as one");
+		maxGap.setToolTipText("Maximum gap between recording periods. Sequential periods with a gap less than this will be counted as one");
 		minLength.setToolTipText("Minimum recording length. Recording sections shorter than this will be ignored");
 		
 		setDialogComponent(mainPanel);
@@ -80,6 +80,7 @@ public class RecordingGapDialog extends PamDialog {
 	@Override
 	public void restoreDefaultSettings() {
 		DeploymentExportOpts defaults = new DeploymentExportOpts();
+		setParams(defaults);
 	}
 
 }

src/tethys/detection/BinnedGranularityHandler.java

@@ -6,11 +6,16 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map.Entry;
+
+import javax.xml.datatype.DatatypeConstants;
+import javax.xml.datatype.XMLGregorianCalendar;
+
 import java.util.Set;
 
 import PamguardMVC.PamDataBlock;
 import PamguardMVC.PamDataUnit;
 import nilus.Detection;
+import nilus.Detections;
 import nilus.SpeciesIDType;
 import tethys.TethysControl;
 import tethys.TethysTimeFuncs;
@@ -54,6 +59,8 @@ public class BinnedGranularityHandler extends GranularityHandler {
 	public void prepare(long timeMillis) {
 //		long binStart = DetectionsHandler.roundDownBinStart(timeMillis, binDurationMillis);
 //		startBin(binStart);
+//		startBin(timeMillis);
+		currentDetections.clear();
 	}
 	
 //	private void startBin(long timeMillis) {
@@ -169,4 +176,9 @@ public class BinnedGranularityHandler extends GranularityHandler {
 		return closeBins(timeMillis);
 	}
 
+	@Override
+	protected boolean autoEffortFix(Detections detections, Detection det) {
+		return contractDetection(detections, det);
+	}
+
 }

src/tethys/detection/CallGranularityHandler.java

@@ -3,6 +3,7 @@ package tethys.detection;
 import PamguardMVC.PamDataBlock;
 import PamguardMVC.PamDataUnit;
 import nilus.Detection;
+import nilus.Detections;
 import tethys.TethysControl;
 import tethys.output.StreamExportParams;
 import tethys.output.TethysExportParams;
@@ -37,4 +38,9 @@ public class CallGranularityHandler extends GranularityHandler {
 		return null;
 	}
 
+	@Override
+	protected boolean autoEffortFix(Detections detections, Detection det) {
+		return expandEffort(detections, det);
+	}
+
 }

src/tethys/detection/DetectionExportProgress.java

@@ -6,10 +6,11 @@ import tethys.niluswraps.PDeployment;
 public class DetectionExportProgress {
 	
 	public static final int STATE_GATHERING = 1;
-	public static final int STATE_CANCELED = 2;
-	public static final int STATE_COMPLETE = 3;
-	public static final int STATE_WRITING = 4;
-	public static final int STATE_COUNTING = 5;
+	public static final int STATE_COUNTING = 2;
+	public static final int STATE_WRITING = 3;
+	public static final int STATE_CANCELED = 4;
+	public static final int STATE_COMPLETE = 5;
+	
 	public PDeployment currentDeployment;
 	public Detections currentDetections;
 	public long lastUnitTime;
@@ -17,12 +18,18 @@ public class DetectionExportProgress {
 	public int exportCount;
 	public int skipCount;
 	public int state;
+	public int totalDeployments, deploymentsDone;
+	public int nMapPoints;
+	public int doneMapPoints;
 	
-	public DetectionExportProgress(PDeployment currentDeployment, Detections currentDetections, long lastUnitTime,
+	public DetectionExportProgress(PDeployment currentDeployment, Detections currentDetections, int nMapPoints, int doneMapPoints, 
+			long lastUnitTime,
 			long totalUnits, int exportCount, int skipCount, int state) {
 		super();
 		this.currentDeployment = currentDeployment;
 		this.currentDetections = currentDetections;
+		this.nMapPoints = nMapPoints;
+		this.doneMapPoints = doneMapPoints;
 		this.lastUnitTime = lastUnitTime;
 		this.totalUnits = totalUnits;
 		this.exportCount = exportCount;

src/tethys/detection/DetectionsHandler.java

@@ -1,9 +1,13 @@
 package tethys.detection;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.ListIterator;
 
 import javax.swing.SwingWorker;
+import javax.xml.datatype.DatatypeConstants;
+import javax.xml.datatype.XMLGregorianCalendar;
 
 import PamController.PamControlledUnit;
 import PamController.PamController;
@@ -32,6 +36,7 @@ import nilus.Detections;
 import nilus.GranularityEnumType;
 import nilus.Helper;
 import tethys.Collection;
+import tethys.CollectionHandler;
 import tethys.TethysControl;
 import tethys.TethysTimeFuncs;
 import tethys.dbxml.DBXMLConnect;
@@ -39,7 +44,6 @@ import tethys.dbxml.TethysException;
 import tethys.deployment.DeploymentHandler;
 import tethys.niluswraps.PDeployment;
 import tethys.niluswraps.PDetections;
-import tethys.output.DatablockSynchInfo;
 import tethys.output.StreamExportParams;
 import tethys.output.TethysExportParams;
 import tethys.pamdata.TethysDataProvider;
@@ -54,9 +58,7 @@ import tethys.swing.export.DetectionsExportWizard;
  * @author dg50
  *
  */
-public class DetectionsHandler {
-
-	private TethysControl tethysControl;
+public class DetectionsHandler extends CollectionHandler {
 
 	public int uniqueDetectionsId=1;
 	public int uniqueDetectionId;
@@ -65,12 +67,14 @@ public class DetectionsHandler {
 
 	private ExportWorker exportWorker;
 
+	public static final String helpPoint = "utilities.tethys.docs.detect_localize";
+
 	/**
 	 * 
 	 * @param tethysControl
 	 */
 	public DetectionsHandler(TethysControl tethysControl) {
-		super();
+		super(tethysControl, Collection.Detections);
 		this.tethysControl = tethysControl;
 	}
 
@@ -296,9 +300,14 @@ public class DetectionsHandler {
 			viewerLoadPolicy = ViewerLoadPolicy.LOAD_UTCNORMAL;
 		}
 		GranularityHandler granularityHandler = GranularityHandler.getHandler(streamExportParams.granularity, tethysControl, dataBlock, exportParams, streamExportParams);
+		int totalMaps = 0;
+		int totalMappedPoints = 0;
+		int totalLoadedDatas = 0;
+		int totalMapPoints = dataMap.getNumMapPoints();
+		int doneMapPoints = 0;
 		for (PDeployment deployment : deployments) {
 			int documentCount = 0;
-			prog = new DetectionExportProgress(deployment, null,
+			prog = new DetectionExportProgress(deployment, null, totalMapPoints, doneMapPoints,
 					lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_COUNTING);
 			exportObserver.update(prog);
 			granularityHandler.prepare(deployment.getAudioStart());
@@ -308,9 +317,10 @@ public class DetectionsHandler {
 			
 			for (OfflineDataMapPoint mapPoint : mapPoints) {
 				if (!activeExport) {
-					prog = new DetectionExportProgress(deployment, null,
+					prog = new DetectionExportProgress(deployment, null,totalMapPoints, doneMapPoints,
 							lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_CANCELED);
 					exportObserver.update(prog);
+					break;
 				}
 
 				if (mapPoint.getEndTime() < deployment.getAudioStart()) {
@@ -319,10 +329,13 @@ public class DetectionsHandler {
 				if (mapPoint.getStartTime() >= deployment.getAudioEnd()) {
 					break;
 				}
+				totalMaps ++;
+				totalMappedPoints += mapPoint.getNDatas();
 				dataBlock.loadViewerData(mapPoint.getStartTime(), mapPoint.getEndTime(), null);
 				ArrayList<PamDataUnit> dataCopy = dataBlock.getDataCopy(deployment.getAudioStart(), deployment.getAudioEnd(), true, dataSelector);
-//				System.out.printf("%d loaded from %s to %s %d kept\n", dataBlock.getUnitsCount(), PamCalendar.formatDateTime(mapPoint.getStartTime()),
-//						PamCalendar.formatDateTime(mapPoint.getEndTime()), dataCopy.size());
+				totalLoadedDatas += dataCopy.size();
+				System.out.printf("%d loaded from %s to %s %d kept\n", dataBlock.getUnitsCount(), PamCalendar.formatDateTime(mapPoint.getStartTime()),
+						PamCalendar.formatDateTime(mapPoint.getEndTime()), dataCopy.size());
 				skipCount += dataBlock.getUnitsCount() - dataCopy.size();
 				for (PamDataUnit dataUnit : dataCopy) {
 					/*
@@ -334,7 +347,7 @@ public class DetectionsHandler {
 						documentCount+=dets.length;
 						
 						if (exportCount % 100 == 0) {
-							prog = new DetectionExportProgress(deployment, null,
+							prog = new DetectionExportProgress(deployment, null,totalMapPoints, doneMapPoints,
 									lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_COUNTING);
 							exportObserver.update(prog);
 						}
@@ -345,15 +358,17 @@ public class DetectionsHandler {
 //					onEffort.getDetection().add(det);
 					lastUnitTime = dataUnit.getTimeMilliseconds();
 				}
-
-				prog = new DetectionExportProgress(deployment, null,
+				doneMapPoints++;
+				prog = new DetectionExportProgress(deployment, null,totalMapPoints, doneMapPoints,
 						lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_COUNTING);
 				exportObserver.update(prog);
 				
 				if (viewerLoadPolicy == ViewerLoadPolicy.LOAD_ALWAYS_EVERYTHING) {
 					break;
 				}
-
+				if (!activeExport) {
+					return 0;
+				}
 			}
 			Detection dets[] = granularityHandler.cleanup(deployment.getAudioEnd());
 			if (dets != null) {
@@ -397,28 +412,33 @@ public class DetectionsHandler {
 		if (viewerLoadPolicy == null) {
 			viewerLoadPolicy = ViewerLoadPolicy.LOAD_UTCNORMAL;
 		}
+		int totalMapPoints = dataMap.getNumMapPoints();
+		int doneMapPoints = 0;
 		GranularityHandler granularityHandler = GranularityHandler.getHandler(streamExportParams.granularity, tethysControl, dataBlock, exportParams, streamExportParams);
 		for (PDeployment deployment : deployments) {
 			int documentCount = 0;
-			prog = new DetectionExportProgress(deployment, null,
+			prog = new DetectionExportProgress(deployment, null,totalMapPoints, doneMapPoints,
 					lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_COUNTING);
 			exportObserver.update(prog);
 			granularityHandler.prepare(deployment.getAudioStart());
 
-			if (currentDetections == null) {
-				currentDetections = startDetectionsDocument(deployment, dataBlock, streamExportParams);
-				currentDetections.getEffort().setStart(TethysTimeFuncs.xmlGregCalFromMillis(deployment.getAudioStart()));
-			}
 			// export everything in that deployment.
 			// need to loop through all map points in this interval.
 			List<OfflineDataMapPoint> mapPoints = dataMap.getMapPoints();
 			for (OfflineDataMapPoint mapPoint : mapPoints) {
 				if (!activeExport) {
-					prog = new DetectionExportProgress(deployment, currentDetections,
+					prog = new DetectionExportProgress(deployment, currentDetections,totalMapPoints, doneMapPoints,
 							lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_CANCELED);
 					exportObserver.update(prog);
+					break;
 				}
 
+				if (currentDetections == null) {
+					// needed in inner loop in case doc gets written at 500000.
+					currentDetections = startDetectionsDocument(deployment, dataBlock, streamExportParams);
+					currentDetections.getEffort().setStart(TethysTimeFuncs.xmlGregCalFromMillis(deployment.getAudioStart()));
+				}
+				
 				if (mapPoint.getEndTime() < deployment.getAudioStart()) {
 					continue;
 				}
@@ -427,6 +447,7 @@ public class DetectionsHandler {
 				}
 				dataBlock.loadViewerData(mapPoint.getStartTime(), mapPoint.getEndTime(), null);
 				ArrayList<PamDataUnit> dataCopy = dataBlock.getDataCopy(deployment.getAudioStart(), deployment.getAudioEnd(), true, dataSelector);
+				Collections.sort(dataCopy);
 				skipCount += dataBlock.getUnitsCount() - dataCopy.size();
 				DetectionGroup onEffort = currentDetections.getOnEffort();
 				for (PamDataUnit dataUnit : dataCopy) {
@@ -442,24 +463,27 @@ public class DetectionsHandler {
 						}
 					}
 					if (exportCount % 100 == 0) {
-						prog = new DetectionExportProgress(deployment, null,
+						prog = new DetectionExportProgress(deployment, currentDetections, totalMapPoints, doneMapPoints,
 								lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_GATHERING);
 						exportObserver.update(prog);
 					}
 					lastUnitTime = dataUnit.getTimeMilliseconds();
 				}
 
-				prog = new DetectionExportProgress(deployment, currentDetections,
+				doneMapPoints ++;
+				prog = new DetectionExportProgress(deployment, currentDetections,totalMapPoints, doneMapPoints,
 						lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_GATHERING);
 				exportObserver.update(prog);
 
-				if (documentCount > 500000 && mapPoint != dataMap.getLastMapPoint()) {
-					prog = new DetectionExportProgress(deployment, currentDetections,
+				if (documentCount > 50000000 && mapPoint != dataMap.getLastMapPoint()) {
+					prog = new DetectionExportProgress(deployment, currentDetections,totalMapPoints, doneMapPoints,
 							lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_WRITING);
 					exportObserver.update(prog);
 					closeDetectionsDocument(currentDetections, mapPoint.getEndTime());
 					try {
-						dbxmlConnect.postAndLog(currentDetections);
+						if (checkDetectionsDocument(currentDetections, granularityHandler)) {
+							dbxmlConnect.postAndLog(currentDetections);
+						}
 					} catch (TethysException e) {
 						tethysControl.showException(e);
 					}
@@ -469,8 +493,14 @@ public class DetectionsHandler {
 				if (viewerLoadPolicy == ViewerLoadPolicy.LOAD_ALWAYS_EVERYTHING) {
 					break;
 				}
+				if (!activeExport) {
+					break;
+				}
+			}
+
+			if (!activeExport) {
+				return DetectionExportProgress.STATE_CANCELED;
 			}
-			
 
 			if (currentDetections != null) {
 				Detection dets[] = granularityHandler.cleanup(deployment.getAudioEnd());
@@ -481,11 +511,13 @@ public class DetectionsHandler {
 						currentDetections.getOnEffort().getDetection().add(dets[dd]);
 					}
 				}
-				prog = new DetectionExportProgress(deployment, currentDetections,
+				prog = new DetectionExportProgress(deployment, currentDetections,totalMapPoints, doneMapPoints,
 						lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_WRITING);
 				closeDetectionsDocument(currentDetections, deployment.getAudioEnd());
 				try {
-					dbxmlConnect.postAndLog(currentDetections);
+					if (checkDetectionsDocument(currentDetections, granularityHandler)) {
+						dbxmlConnect.postAndLog(currentDetections);
+					}
 				} catch (TethysException e) {
 					tethysControl.showException(e);
 				}
@@ -493,7 +525,7 @@ public class DetectionsHandler {
 			}
 		}
 
-		prog = new DetectionExportProgress(null, null,
+		prog = new DetectionExportProgress(null, null,totalMapPoints, totalMapPoints,
 				lastUnitTime, totalCount, exportCount, skipCount, DetectionExportProgress.STATE_COMPLETE);
 		exportObserver.update(prog);
 		return DetectionExportProgress.STATE_COMPLETE;
@@ -552,7 +584,7 @@ public class DetectionsHandler {
 		supportSoft.setVersion(getSupportSoftwareVersion(dataBlock));
 		supSoft.add(supportSoft);
 		detections.setAlgorithm(algorithm);
-		detections.setUserId("Unknown user");
+		detections.setUserId("PAMGuard user");
 		detections.setEffort(getDetectorEffort(deployment, dataBlock, exportParams));
 
 		return detections;
@@ -567,6 +599,50 @@ public class DetectionsHandler {
 	private void closeDetectionsDocument(Detections detections, Long audioEnd) {
 		detections.getEffort().setEnd(TethysTimeFuncs.xmlGregCalFromMillis(audioEnd));
 	}
+	
+	/**
+	 * Run some checks on the Detections document prior to submission. <br>
+	 * Currently, is is just a check that the detections are within the effort times.   
+	 * @param detections Detections document
+	 * @return false if there is an outstanding problem. 
+	 */
+	private boolean checkDetectionsDocument(Detections detections, GranularityHandler granularityHandler) {
+		XMLGregorianCalendar effStart = detections.getEffort().getStart();
+		XMLGregorianCalendar effEnd = detections.getEffort().getEnd();
+		DetectionGroup dets = detections.getOnEffort();
+		List<Detection> detList = dets.getDetection();
+		ListIterator<Detection> detIt = detList.listIterator();
+		while (detIt.hasNext()) {
+			Detection det = detIt.next();
+			XMLGregorianCalendar detS = det.getStart();
+			XMLGregorianCalendar detE = det.getEnd();
+			if (effStart.compare(detS) == DatatypeConstants.GREATER) {
+				if (granularityHandler.autoEffortFix(detections, det)) {
+					continue;
+				}
+				String str = String.format("<html>A Detection at %s starts before the document effort start at %s<br>"
+						+ "Do you want to adjust the effort start time or abort export ?</html>", detS, effStart);
+				int ans = WarnOnce.showNamedWarning("TETHYSDETNOTINEFFORT", tethysControl.getGuiFrame(), "Detection Document Warning", str, WarnOnce.OK_CANCEL_OPTION);
+				if (ans == WarnOnce.CANCEL_OPTION) {
+					return false;
+				}
+				detections.getEffort().setStart(detS);
+			}
+			if (effEnd.compare(detE) == DatatypeConstants.LESSER) {
+				if (granularityHandler.autoEffortFix(detections, det)) {
+					continue;
+				}
+				String str = String.format("<html>A Detection at %s-%s ends <br>after the document effort end at %s<br>"
+						+ "Do you want to adjust the effort end time or abort export ?</html>", detS, detE, effStart);
+				int ans = WarnOnce.showNamedWarning("TETHYSDETNOTINEFFORT", tethysControl.getGuiFrame(), "Detection Document Warning", str, WarnOnce.OK_CANCEL_OPTION);
+				if (ans == WarnOnce.CANCEL_OPTION) {
+					return false;
+				}
+				detections.getEffort().setEnd(detE);
+			}
+		}
+		return true;
+	}
 
 	/**
 	 * Worker thread for exporting detections. 
@@ -599,13 +675,16 @@ public class DetectionsHandler {
 		protected Integer doInBackground() throws Exception {
 			Integer ans = null;
 			try {
-				int count = countDetections(dataBlock, exportParams, exportObserver);
-				String msg = String.format("Do you want to go ahead and output %d %s detections to Tethys?", 
-						count, exportParams.granularity);
-				int doit = WarnOnce.showWarning("Tethys Detections Export", msg, WarnOnce.OK_CANCEL_OPTION);
-				if (doit == WarnOnce.OK_OPTION) {
+//				int count = countDetections(dataBlock, exportParams, exportObserver);
+//				if (activeExport == false) {
+//					return 0;
+//				}
+//				String msg = String.format("Do you want to go ahead and output %d %s detections to Tethys?", 
+//						count, exportParams.granularity);
+//				int doit = WarnOnce.showWarning("Tethys Detections Export", msg, WarnOnce.OK_CANCEL_OPTION);
+//				if (doit == WarnOnce.OK_OPTION) {
 					ans = exportDetections(dataBlock, exportParams, this);
-				}
+//				}
 			}
 			catch (Exception e) {
 				e.printStackTrace();
@@ -616,7 +695,7 @@ public class DetectionsHandler {
 		@Override
 		protected void done() {
 //			this.
-			DetectionExportProgress prog = new DetectionExportProgress(null, null, 0, 0, 0, 0, DetectionExportProgress.STATE_COMPLETE);
+			DetectionExportProgress prog = new DetectionExportProgress(null, null, 0, 0,  0, 0, 0, 0, DetectionExportProgress.STATE_COMPLETE);
 			tethysControl.exportedDetections(dataBlock);
 			exportObserver.update(prog);
 			TethysReporter.getTethysReporter().showReport(tethysControl.getGuiFrame(), true);
@@ -661,4 +740,10 @@ public class DetectionsHandler {
 		DetectionsExportWizard.showDialog(tethysControl.getGuiFrame(), tethysControl, dataBlock);
 
 	}
+
+
+	@Override
+	public String getHelpPoint() {
+		return helpPoint;
+	}
 }

src/tethys/detection/EncounterGranularityHandler.java

@@ -11,6 +11,7 @@ import java.util.Map.Entry;
 import PamguardMVC.PamDataBlock;
 import PamguardMVC.PamDataUnit;
 import nilus.Detection;
+import nilus.Detections;
 import nilus.SpeciesIDType;
 import tethys.TethysControl;
 import tethys.TethysTimeFuncs;
@@ -48,7 +49,7 @@ public class EncounterGranularityHandler extends GranularityHandler {
 
 	@Override
 	public void prepare(long timeMillis) {
-
+		currentDetections.clear();
 	}
 
 	@Override
@@ -77,6 +78,7 @@ public class EncounterGranularityHandler extends GranularityHandler {
 			currentDetections.put(groupName, det);
 		}
 		else {
+			
 			// add to current detection. Set new end time and increment count
 			det.setEnd(TethysTimeFuncs.xmlGregCalFromMillis(dataUnit.getEndTimeInMilliseconds()));
 			int count = det.getCount().intValue() + 1;
@@ -122,34 +124,17 @@ public class EncounterGranularityHandler extends GranularityHandler {
 		}
 	}
 
-	//	private Detection[] checkCurrentEncounters(long timeMilliseconds) {
-	//		if (currentDetections == null || currentDetections.size() == 0) {
-	//			return null;
-	//		}
-	//		int nGood = 0;
-	//		Detection[] newDetections = new Detection[currentDetections.size()];
-	//		Iterator<Detection> detIt = currentDetections.iterator();
-	//		while (detIt.hasNext()) {
-	//			Detection aDet = detIt.next();
-	//			Long detEnd = TethysTimeFuncs.millisFromGregorianXML(aDet.getEnd());
-	//			if (timeMilliseconds-detEnd > maxGapMillis) {
-	//				detIt.remove();
-	//				newDetections[nGood++] = aDet;
-	//			}
-	//		}
-	//		
-	//		if (nGood == 0) {
-	//			return null;
-	//		}
-	//		else {
-	//			return Arrays.copyOf(newDetections, nGood);
-	//		}
-	//	}
 
 	@Override
 	public Detection[] cleanup(long timeMillis) {
 		// get everything still on the go. 
-		return checkCurrentEncounters(timeMillis + maxGapMillis);
+		return checkCurrentEncounters(timeMillis + maxGapMillis*10);
 	}
 
+	@Override
+	protected boolean autoEffortFix(Detections detections, Detection det) {
+		return expandEffort(detections, det);
+	}
+
+
 }

src/tethys/detection/GranularityHandler.java

@@ -1,8 +1,16 @@
 package tethys.detection;
 
+import java.util.List;
+import java.util.ListIterator;
+
+import javax.xml.datatype.DatatypeConstants;
+import javax.xml.datatype.XMLGregorianCalendar;
+
 import PamguardMVC.PamDataBlock;
 import PamguardMVC.PamDataUnit;
 import nilus.Detection;
+import nilus.DetectionGroup;
+import nilus.Detections;
 import nilus.GranularityEnumType;
 import tethys.TethysControl;
 import tethys.output.StreamExportParams;
@@ -118,4 +126,93 @@ public abstract class GranularityHandler {
 		}
 		return null;
 	}
+
+	/**
+	 * Automatically fix mismatches between effort and detections. This will be called if a 
+	 * detection or part of a detection is outside of the start and end defined by the effort. If it's a 
+	 * small difference, i.e. if the detection at least overlaps the effort then it can be automatically 
+	 * fixed by truncating the detection (for binned types) or by a small extension to the effort (for encounter
+	 * and call types).  
+	 * @param detections nilus Detections object 
+	 * @param det a single detection
+	 * @return true if it was fixed automatically. False otherwise. 
+	 */
+	protected abstract boolean autoEffortFix(Detections detections, Detection det);
+	
+	/**
+	 * Check that the detection at least overlaps the effort period. 
+	 * @param detections nilus Detections object 
+	 * @param det a single detection
+	 * @return true if the overlap
+	 */
+	protected boolean effortOverlap(Detections detections, Detection det) {
+		XMLGregorianCalendar effStart = detections.getEffort().getStart();
+		XMLGregorianCalendar effEnd = detections.getEffort().getEnd();
+		XMLGregorianCalendar detStart = det.getStart();
+		XMLGregorianCalendar detEnd = det.getEnd();
+		if (effStart.compare(detEnd) == DatatypeConstants.GREATER) {
+			return false;
+		}
+		if (effEnd.compare(detStart) == DatatypeConstants.LESSER) {
+			return false;
+		}
+		return true;
+	}
+
+	/**
+	 * Fix effort / detection problem but contracting the start / end times of the detection
+	 * @param detections nilus Detections object 
+	 * @param det a single detection
+	 * @return true if fixed automatically
+	 */
+	protected boolean contractDetection(Detections detections, Detection det) {
+		if (effortOverlap(detections, det) == false) {
+			return false;
+		}
+		// at least some overlap, so fix it.
+		// going to fix it my shortening the detection, and leave the effort alone. 
+		XMLGregorianCalendar effStart = detections.getEffort().getStart();
+		XMLGregorianCalendar effEnd = detections.getEffort().getEnd();
+		XMLGregorianCalendar detStart = det.getStart();
+		XMLGregorianCalendar detEnd = det.getEnd();
+		
+
+		if (effStart.compare(detStart) == DatatypeConstants.GREATER) {
+			System.out.printf("Fix Detections change detection start from %s to %s\n", detStart, effStart);
+			det.setStart(effStart);
+		}
+		if (effEnd.compare(detEnd) == DatatypeConstants.LESSER) {
+			System.out.printf("Fix Detections change detection end from %s to %s\n", detEnd, effEnd);
+			det.setEnd(effEnd);
+		}
+		return true;
+	}
+
+	/**
+	 * Fix effort / detection problem but expanding the start / end times of the effort
+	 * @param detections nilus Detections object 
+	 * @param det a single detection
+	 * @return true if fixed automatically
+	 */
+	protected boolean expandEffort(Detections detections, Detection det) {
+		if (effortOverlap(detections, det) == false) {
+			return false;
+		}
+		// at least some overlap, so fix it.
+		// going to fix it my shortening the detection, and leave the effort alone. 
+		XMLGregorianCalendar effStart = detections.getEffort().getStart();
+		XMLGregorianCalendar effEnd = detections.getEffort().getEnd();
+		XMLGregorianCalendar detStart = det.getStart();
+		XMLGregorianCalendar detEnd = det.getEnd();
+		
+		if (effStart.compare(detStart) == DatatypeConstants.GREATER) {
+			System.out.printf("Fix Detections change effort start from %s to %s\n", effStart, detStart);
+			detections.getEffort().setStart(detStart);
+		}
+		if (effEnd.compare(detEnd) == DatatypeConstants.LESSER) {
+			System.out.printf("Fix Detections change effort end from %s to %s\n", effEnd, detEnd);
+			detections.getEffort().setEnd(detEnd);
+		}
+		return true;
+	}
 }

src/tethys/localization/LocalizationHandler.java

@@ -0,0 +1,17 @@
+package tethys.localization;
+
+import nilus.CylindricalCoordinateType;
+import nilus.LocalizationType;
+import nilus.Localize.Effort.CoordinateReferenceSystem;
+
+public class LocalizationHandler {
+
+	
+	public LocalizationType getLoc() {
+		LocalizationType lt = new LocalizationType();
+		CylindricalCoordinateType cct = new CylindricalCoordinateType();
+//		cct.set
+		CoordinateReferenceSystem cr;
+		return null;
+	}
+}

src/tethys/niluswraps/NilusChecker.java

@@ -56,7 +56,7 @@ public class NilusChecker {
 		for (Field f : emptyFields) {
 			msg += String.format("<br>Field %s in object %s", f.getName(), f.getDeclaringClass().getName());
 		}
-		msg += "<br><br>It is likely that this document will fail to write to the Tethys database.</html>";
+		msg += "<br><br>It is possible that this document will fail to write to the Tethys database.</html>";
 		String tit = "Incomplete Tethys data";
 		WarnOnce.showWarning(owner, tit, msg, WarnOnce.WARNING_MESSAGE);
 		return false;

src/tethys/niluswraps/NilusSettingsWrapper.java

@@ -129,7 +129,9 @@ public class NilusSettingsWrapper<T extends Object> implements Serializable, Clo
 			Document doc = builder.parse(new InputSource(new StringReader(xmlString)));
 			return doc;
 		} catch (Exception e) {
-			e.printStackTrace();
+			System.out.println(e.getMessage());
+			System.out.println("Nilus Settings wrapper - Error parsing string\n" + xmlString);
+//			e.printStackTrace();
 		}
 		return null;
 	}

src/tethys/output/StreamExportParams.java

@@ -98,7 +98,11 @@ public class StreamExportParams implements Serializable {
 	 * @return
 	 */
 	public DescriptionType getNilusDetectionDescription() {
-		return getDetectionDescription().getDescription();
+		WrappedDescriptionType desc = getDetectionDescription();
+		if (desc == null) {
+			return null;
+		}
+		return desc.getDescription();
 	}
 
 }

src/tethys/output/TethysExportParams.java

@@ -45,6 +45,8 @@ public class TethysExportParams implements Serializable, Cloneable{
 	private String datasetName;
 
 	public boolean listDocsInPamguard;
+
+	private String effortSourceName;
 	
 
 	/**
@@ -121,6 +123,22 @@ public class TethysExportParams implements Serializable, Cloneable{
 		return streamParamsMap.get(longDataName);
 	}
 
+	/**
+	 * Source name for type of effort. 
+	 * @param sourceName
+	 */
+	public void setEffortSourceName(String sourceName) {
+		this.effortSourceName = sourceName;
+	}
+
+	/**
+	 * Source name for type of effort. 
+	 * @return the effortSourceName
+	 */
+	public String getEffortSourceName() {
+		return effortSourceName;
+	}
+
 
 
 }

src/tethys/pamdata/AutoTethysProvider.java

@@ -286,6 +286,7 @@ abstract public class AutoTethysProvider implements TethysDataProvider {
 			detParams.setMaxFreqHz(freqs[1]);
 		}
 		double ampli = dataUnit.getAmplitudeDB();
+		ampli = roundDecimalPlaces(ampli, 1);
 		detParams.setReceivedLevelDB(ampli);
 		//		DataUnitBaseData basicData = dataUnit.getBasicData();
 		gotTonalContour(dataUnit, detParams);

src/tethys/species/DataBlockSpeciesManager.java

@@ -97,7 +97,10 @@ abstract public class DataBlockSpeciesManager<T extends PamDataUnit> {
 	public SpeciesMapItem getSpeciesItem(T dataUnit) {
 		String speciesString = getSpeciesCode(dataUnit);
 		if (speciesString == null) {
-			return getDefaultDefaultSpecies();
+			SpeciesMapItem def = getDefaultDefaultSpecies();
+			if (def != null) {
+				speciesString = def.getPamguardName();
+			}
 		}
 		DataBlockSpeciesMap speciesMap = getDatablockSpeciesMap();
 		if (speciesMap == null) {
@@ -127,7 +130,26 @@ abstract public class DataBlockSpeciesManager<T extends PamDataUnit> {
 		if (allCodes.size() == 0) {
 			allCodes.add("Unknown");
 		}
-		return allCodes;
+		return makeUniqueList(allCodes);
+	}
+	
+	/**
+	 * Make sure all entries in an array list are unique. 
+	 * @param list
+	 * @return updated list. 
+	 */
+	public ArrayList<String> makeUniqueList(ArrayList<String> list) {
+		if (list == null) {
+			return null;
+		}
+		ArrayList<String> newList = new ArrayList();
+		for (String aStr : list) {
+			if (newList.contains(aStr)) {
+				continue;
+			}
+			newList.add(aStr);
+		}
+		return newList;
 	}
 	
 	public DataBlockSpeciesMap getDatablockSpeciesMap() {

src/tethys/species/ITISFunctions.java

@@ -70,7 +70,7 @@ public class ITISFunctions {
 //		PAMGuardXMLPreview xmlPreview = new PAMGuardXMLPreview(null, "returned", qResult.queryResult)
 		PamguardXMLWriter pamXMLWriter = PamguardXMLWriter.getXMLWriter();
 		String fDoc = pamXMLWriter.getAsString(doc, true);
-		System.out.println(fDoc);
+//		System.out.println(fDoc);
 		
 		String tsn = dbQueries.getElementData(docEl, "tsn");
 		if (tsn == null) {
@@ -91,7 +91,46 @@ public class ITISFunctions {
 		return new TethysITISResult(itisCode, taxunit, latin, vernacular);
 	}
 	
+	/**
+	 * Search species codes. If the search term is a valid Integer number
+	 * then it's assumed to be an ITIS code and the function should
+	 * return a single map item. If it's non-integer, it's assumed to 
+	 * be a common or latin name search
+	 * @param searchTerm
+	 * @return array list of possible matches. 
+	 */
 	public ArrayList<SpeciesMapItem> searchSpecies(String searchTerm) {
+		Integer intVal = null;
+		try {
+			intVal = Integer.valueOf(searchTerm);
+		}
+		catch (NumberFormatException e) {
+			intVal = null;
+		}
+		if (intVal != null) {
+			return searchCodes(intVal);
+		}
+		else { // assume name search
+			return searchNames(searchTerm);
+		}
+	}
+	
+	private ArrayList<SpeciesMapItem> searchCodes(Integer intCode) {
+		ArrayList<SpeciesMapItem> mapItems = new ArrayList();
+		TethysITISResult result = getITISInformation(intCode);
+		if (result != null) {
+			mapItems.add(new SpeciesMapItem(intCode, "", "", result.getLatin(), result.getVernacular()));
+		}
+		return mapItems;
+	}
+
+	/**
+	 * Search common and latin names for partial matched of the search term
+	 * and return an array list of all possible matches. 
+	 * @param searchTerm
+	 * @return
+	 */
+	public ArrayList<SpeciesMapItem> searchNames(String searchTerm) {
 		ArrayList<SpeciesMapItem> items = new ArrayList<SpeciesMapItem>();		
 		String xQ = "let $target := \"thespeciessearchterm\" \r\n"
 				+ "return\r\n"

src/tethys/species/ITISTypes.java

@@ -7,8 +7,8 @@ package tethys.species;
  */
 public class ITISTypes {
 
-	public static final int OTHER = 0;
-	public static final int ANTHROPOGENIC = 1;
+	public static final int OTHER = -10;
+	public static final int ANTHROPOGENIC = 1758;
 	
 	public static final String getName(int code) {
 		switch (code) {

src/tethys/species/SpeciesMapItem.java

@@ -41,6 +41,14 @@ public class SpeciesMapItem implements Serializable, Cloneable {
 	 */
 	private String callType;
 
+	/**
+	 * 
+	 * @param itisCode
+	 * @param callType
+	 * @param pamguardName
+	 * @param latinName
+	 * @param commonName
+	 */
 	public SpeciesMapItem(int itisCode, String callType, String pamguardName, String latinName, String commonName) {
 		super();
 		this.itisCode = itisCode;
@@ -50,6 +58,12 @@ public class SpeciesMapItem implements Serializable, Cloneable {
 		this.commonName = commonName;
 	}
 
+	/**
+	 * 
+	 * @param itisCode
+	 * @param callType
+	 * @param pamguardName
+	 */
 	public SpeciesMapItem(int itisCode, String callType, String pamguardName) {
 		super();
 		this.itisCode = itisCode;

src/tethys/species/swing/DataBlockSpeciesDialog.java

@@ -49,16 +49,7 @@ public class DataBlockSpeciesDialog extends PamDialog {
 				+ "\"Other Phenomena\" (-10).  "
 				+ "<br>When known, a call or sound type should "
 				+ "be specified (see help for more information).</html>";
-		nPanel.add(BorderLayout.CENTER, new JLabel(otherMsg , JLabel.LEFT));
-//		JPanel nwBit = new JPanel(new FlowLayout());
-//		JButton exportButton = new JButton("Export");
-//		exportButton.addActionListener(SpeciesMapManager.getInstance().getExportAction(parentFrame));
-//		nwBit.add(exportButton);
-//		JButton importButton = new JButton("Import");
-//		importButton.addActionListener(SpeciesMapManager.getInstance().getImportAction(parentFrame));
-//		nwBit.add(importButton);
-//		nPanel.add(BorderLayout.WEST, nwBit);
-		
+		nPanel.add(BorderLayout.CENTER, new JLabel(otherMsg , JLabel.LEFT));		
 		
 		mainPanel.add(BorderLayout.NORTH, nPanel);
 		setDialogComponent(mainPanel);

src/tethys/species/swing/SpeciesSearchDialog.java

@@ -104,11 +104,11 @@ public class SpeciesSearchDialog extends PamDialog {
 		setResizable(true);
 		setDialogComponent(mainPanel);
 	}
-	public static SpeciesMapItem showDialog(Window parentFrame, TethysControl tethysControl) {
+	public static SpeciesMapItem showDialog(Window parentFrame, TethysControl tethysControl, Integer currentCode) {
 		if (singleInstance == null) {
 			singleInstance = new SpeciesSearchDialog(parentFrame, tethysControl);
 		}
-		singleInstance.setParams();
+		singleInstance.setParams(currentCode);
 		singleInstance.setVisible(true);
 		return singleInstance.selectedItem;
 	}
@@ -131,6 +131,9 @@ public class SpeciesSearchDialog extends PamDialog {
 	
 	public void setMapItems(ArrayList<SpeciesMapItem> newMapItems) {
 		this.speciesMapItems = newMapItems;
+		if (newMapItems != null && newMapItems.size() == 1) {
+			setSelectedItem(newMapItems.get(0));
+		}
 		tableModel.fireTableDataChanged();
 	}
 
@@ -188,14 +191,20 @@ public class SpeciesSearchDialog extends PamDialog {
 		
 	}
 	
-	private void setParams() {
-		searchText.setText(null);
-		clearResults();
+	private void setParams(Integer currentCode) {
+		if (currentCode == null) {
+			searchText.setText(null);
+			clearResults();
+		}
+		else {
+			searchText.setText(currentCode.toString());
+			searchTethys();
+		}
 	}
 
 	private void clearResults() {
 		speciesMapItems = null;
-		selectedItem = null;
+		setSelectedItem(null);
 	}
 	@Override
 	public boolean getParams() {
@@ -216,6 +225,10 @@ public class SpeciesSearchDialog extends PamDialog {
 
 	}
 	
+	private void enableControls() {
+		getOkButton().setEnabled(selectedItem != null);
+	}
+	
 	private class TableMouse extends MouseAdapter {
 
 		@Override
@@ -225,12 +238,18 @@ public class SpeciesSearchDialog extends PamDialog {
 			}
 			int selectedRow = resultTable.getSelectedRow();
 			if (selectedRow >= 0 && selectedRow < speciesMapItems.size()) {
-				selectedItem = speciesMapItems.get(selectedRow);
+				setSelectedItem(speciesMapItems.get(selectedRow));
 			}
 			tableModel.fireTableDataChanged();
 		}
 		
 	}
+	
+	private void setSelectedItem(SpeciesMapItem selItem) {
+		this.selectedItem = selItem;
+		enableControls();
+	}
+	
 	private class DataModel extends AbstractTableModel {
 		
 		private String[] colNames = {"Select", "TSN", "Name", "Common Name"};

src/tethys/species/swing/SpeciesSubPanel.java

@@ -114,12 +114,12 @@ public class SpeciesSubPanel {
 		}
 		ITISFunctions itisFunctions = tethysControl.getItisFunctions();
 		String itisString = this.itisCode.getText();
-		if (itisString == null || itisString.length() == 0) {
+//		if (itisString == null || itisString.length() == 0) {
 			searchForCode(tethysControl, itisFunctions);
-		}
-		else {
-			getCodeInformation(tethysControl, itisFunctions, itisString);
-		}
+//		}
+//		else {
+//			getCodeInformation(tethysControl, itisFunctions, itisString);
+//		}
 //		System.out.println(itisInfo);
 	}
 
@@ -144,7 +144,15 @@ public class SpeciesSubPanel {
 	}
 
 	private void searchForCode(TethysControl tethysControl, ITISFunctions itisFunctions) {
-		SpeciesMapItem speciesItem = SpeciesSearchDialog.showDialog(tethysControl.getGuiFrame(), tethysControl);
+		Integer currentCode = null;
+		try {
+			currentCode = Integer.valueOf(itisCode.getText());
+		}
+		catch (NumberFormatException e) {
+			
+		}
+		
+		SpeciesMapItem speciesItem = SpeciesSearchDialog.showDialog(tethysControl.getGuiFrame(), tethysControl, currentCode);
 		if (speciesItem != null) {
 			itisCode.setText(String.format("%d", speciesItem.getItisCode()));
 			latinName.setText(speciesItem.getLatinName());

src/tethys/swing/DatablockDetectionsPanel.java

@@ -5,6 +5,7 @@ import java.awt.event.ActionEvent;
 import java.awt.event.ActionListener;
 import java.awt.event.MouseAdapter;
 import java.awt.event.MouseEvent;
+import java.util.ArrayList;
 import java.util.List;
 
 import javax.swing.JComponent;
@@ -19,6 +20,8 @@ import javax.swing.table.AbstractTableModel;
 import javax.swing.table.JTableHeader;
 import javax.xml.datatype.XMLGregorianCalendar;
 
+import PamUtils.worker.PamWorkWrapper;
+import PamUtils.worker.PamWorker;
 import PamView.PamGui;
 import PamView.dialog.warn.WarnOnce;
 import PamView.tables.SwingTableColumnWidths;
@@ -42,7 +45,7 @@ import tethys.niluswraps.PDetections;
  * @author dg50
  *
  */
-public class DatablockDetectionsPanel extends TethysGUIPanel implements StreamTableObserver {
+public class DatablockDetectionsPanel extends TethysGUIPanel implements StreamTableObserver, PamWorkWrapper<String> {
 
 	private JPanel mainPanel;
 
@@ -118,15 +121,38 @@ public class DatablockDetectionsPanel extends TethysGUIPanel implements StreamTa
 
 	@Override
 	public void selectDataBlock(PamDataBlock dataBlock) {
+		if (this.dataBlock == dataBlock) {
+			return; // stops lots of requerying, which matters when database is large. 
+		}
 		this.dataBlock = dataBlock;
-		dataBlockName.setText(dataBlock.getLongDataName());
-		streamDetectionsSummary = getTethysControl().getDetectionsHandler().getStreamDetections(dataBlock);
+		if (dataBlock == null) {
+			dataBlockName.setText("Select data in panel on the left");
+			return;
+		}
+		else {
+			dataBlockName.setText(dataBlock.getLongDataName());
+		}
+		// need to re-thread this to stop user panicing that nothing is happening. 
+		PamWorker w = new PamWorker<String>(this, getTethysControl().getGuiFrame(), 0, "Searching database for " + dataBlock.getDataName());
+		w.start();
+	}
+
+	@Override
+	public void taskFinished(String result) {
 		tableModel.fireTableDataChanged();
 	}
 
+	@Override
+	public String runBackgroundTask(PamWorker<String> pamWorker) {
+		streamDetectionsSummary = getTethysControl().getDetectionsHandler().getStreamDetections(dataBlock);
+		return null;
+	}
+
 	@Override
 	public void updateState(TethysState tethysState) {
 		if (dataBlock != null) {
+			PamDataBlock currBlock = dataBlock;
+			selectDataBlock(null);
 			selectDataBlock(dataBlock);
 		}
 	}
@@ -164,16 +190,9 @@ public class DatablockDetectionsPanel extends TethysGUIPanel implements StreamTa
 
 		JPopupMenu popMenu = new JPopupMenu();
 
+		JMenuItem menuItem;
 		if (rows.length == 1) {
-			JMenuItem menuItem = new JMenuItem("Delete document " + pDets.detections.getId());
-			menuItem.addActionListener(new ActionListener() {
-				@Override
-				public void actionPerformed(ActionEvent e) {
-					deleteDocument(pDets);
-				}
-			});
-			popMenu.add(menuItem);
-
+			
 			menuItem = new JMenuItem("Display document " + pDets.detections.getId());
 			menuItem.addActionListener(new ActionListener() {
 				@Override
@@ -191,9 +210,19 @@ public class DatablockDetectionsPanel extends TethysGUIPanel implements StreamTa
 				}
 			});
 			popMenu.add(menuItem);
+			
+			popMenu.addSeparator();
+			menuItem = new JMenuItem("Delete document " + pDets.detections.getId());
+			menuItem.addActionListener(new ActionListener() {
+				@Override
+				public void actionPerformed(ActionEvent e) {
+					deleteDocument(pDets);
+				}
+			});
+			popMenu.add(menuItem);
 		}
 		else if (rows.length > 0){
-			JMenuItem menuItem = new JMenuItem("Delete multiple Detections documents");
+			menuItem = new JMenuItem("Delete multiple Detections documents");
 			menuItem.addActionListener(new ActionListener() {
 
 				@Override
@@ -217,20 +246,50 @@ public class DatablockDetectionsPanel extends TethysGUIPanel implements StreamTa
 		if (ans != WarnOnce.OK_OPTION) {
 			return;
 		}
+
+		ArrayList<Detections> toDelete = new ArrayList();
+
 		for (int i = 0; i < rows.length; i++) {
 			int row = rows[i];
 			PDetections pDets = detectionsForRow(row);
 			if (pDets == null) {
 				continue;
 			}
-			try {
-				getTethysControl().getDbxmlConnect().deleteDocument(pDets.detections);
-			} catch (TethysException e) {
-				getTethysControl().showException(e);
-			}
+			toDelete.add(pDets.detections);
 		}
-		getTethysControl().exportedDetections(dataBlock);
-		selectDataBlock(dataBlock); // force table update. 
+		DeleteDocs dd = new DeleteDocs(toDelete);
+		PamWorker<Integer> worker = new PamWorker(dd, getTethysControl().getGuiFrame(), 1, "Deleting Detections documents");
+		worker.start();
+
+	}
+
+	private class DeleteDocs implements PamWorkWrapper<Integer> {
+
+		private ArrayList<Detections> toDelete;
+
+		public DeleteDocs(ArrayList<Detections> toDelete) {
+			this.toDelete = toDelete;
+		}
+
+		@Override
+		public Integer runBackgroundTask(PamWorker<Integer> pamWorker) {
+			for (Detections dets : toDelete) {
+				try {
+					
+					getTethysControl().getDbxmlConnect().deleteDocument(dets);
+				} catch (TethysException e) {
+					getTethysControl().showException(e);
+				}
+			}
+			return toDelete.size();
+		}
+
+		@Override
+		public void taskFinished(Integer result) {
+			getTethysControl().exportedDetections(dataBlock);
+			selectDataBlock(dataBlock); // force table update. 			
+		}
+
 	}
 
 	protected void deleteDocument(PDetections pDets) {

src/tethys/swing/DatablockSynchPanel.java

@@ -1,6 +1,7 @@
 package tethys.swing;
 
 import java.awt.BorderLayout;
+import java.awt.Color;
 import java.awt.event.ActionEvent;
 import java.awt.event.ActionListener;
 import java.awt.event.ComponentEvent;
@@ -13,6 +14,7 @@ import java.util.ArrayList;
 
 import javax.swing.JButton;
 import javax.swing.JComponent;
+import javax.swing.JLabel;
 import javax.swing.JMenuItem;
 import javax.swing.JPanel;
 import javax.swing.JPopupMenu;
@@ -36,9 +38,9 @@ import tethys.niluswraps.PDeployment;
 import tethys.output.DatablockSynchInfo;
 import tethys.species.DataBlockSpeciesManager;
 
-public class DatablockSynchPanel extends TethysGUIPanel {
+public class DatablockSynchPanel extends TethysExportPanel {
 	
-	public JPanel mainPanel;
+//	public JPanel mainPanel;
 	
 	private JTable synchTable;
 	
@@ -48,39 +50,46 @@ public class DatablockSynchPanel extends TethysGUIPanel {
 	
 	private ArrayList<StreamTableObserver> tableObservers = new ArrayList<>();
 	
-	private JButton exportButton;
+//	private TippedButton exportButton;
 
+//	private JLabel exportWarning;
+	
+	
 	public DatablockSynchPanel(TethysControl tethysControl) {
-		super(tethysControl);
-		mainPanel = new PamPanel(new BorderLayout());
+		super(tethysControl, tethysControl.getDetectionsHandler(), false);
+//		mainPanel = new PamPanel(new BorderLayout());
+		JPanel mainPanel = getMainPanel();
 		mainPanel.setBorder(new TitledBorder("PAMGuard data blocks"));
 		synchTableModel = new SynchTableModel();
 		synchTable = new JTable(synchTableModel);
 		new SwingTableColumnWidths(tethysControl.getUnitName()+"SynchTable", synchTable);
 		JScrollPane scrollPane = new JScrollPane(synchTable);
 		mainPanel.add(BorderLayout.CENTER, scrollPane);
-		PamPanel ctrlPanel = new PamPanel(new BorderLayout());
-		exportButton = new JButton("Export ...");
-		ctrlPanel.add(BorderLayout.WEST, exportButton);
-		mainPanel.add(BorderLayout.NORTH, ctrlPanel);
+//		PamPanel ctrlPanel = new PamPanel(new BorderLayout());
+//		exportButton = new TippedButton("Export ...", "Export Detections document");
+//		exportWarning = new JLabel("  ");
+//		exportWarning.setForeground(Color.RED);
+//		ctrlPanel.add(BorderLayout.WEST, exportButton);
+//		ctrlPanel.add(BorderLayout.CENTER, exportWarning);
+//		mainPanel.add(BorderLayout.NORTH, ctrlPanel);
 
 		
 		synchTable.addMouseListener(new MouseActions());
 		synchTable.addKeyListener(new KeyActions());
 		
-		exportButton.addActionListener(new ActionListener() {
-			@Override
-			public void actionPerformed(ActionEvent e) {
-				exportData();
-			}
-		});
+//		exportButton.addActionListener(new ActionListener() {
+//			@Override
+//			public void actionPerformed(ActionEvent e) {
+//				exportData();
+//			}
+//		});
 		enableExportButton();
 	}
 
-	@Override
-	public JComponent getComponent() {
-		return mainPanel;
-	}
+//	@Override
+//	public JComponent getComponent() {
+//		return mainPanel;
+//	}
 	
 	private class KeyActions extends KeyAdapter {
 		@Override
@@ -141,13 +150,50 @@ public class DatablockSynchPanel extends TethysGUIPanel {
 	}
 
 	private void enableExportButton() {
+		if (!getTethysControl().isServerOk()) {
+			disableExport("Tethys Server not running");
+			return;
+		}
 		int[] rows = synchTable.getSelectedRows();
-		boolean en = rows != null && rows.length == 1;
 		ArrayList<PDeployment> deployments = getTethysControl().getDeploymentHandler().getMatchedDeployments();
 		if (deployments == null || deployments.size() == 0) {
-			en = false;
+			disableExport("No Deployment document(s). Export Deployments prior to exporting Detections");
+			return;
 		}
-		exportButton.setEnabled(getTethysControl().isServerOk() & en);
+		boolean en = rows != null && rows.length == 1;
+		if (!en) {
+			disableExport("No PAMGuard datablock selected (click a row on the table below)");
+			return;
+		}
+
+		PamDataBlock dataBlock = dataBlockSynchInfo.get(rows[0]).getDataBlock();
+		String mapError = checkSpeciesManager(dataBlock);
+		if (mapError != null) {
+			disableExport("Unable to export due to species map error: " + mapError + ". Right click table row to edit species list");
+			return;
+		}
+		
+		enableExport(true);
+	}
+	
+//	public void disableExport(String reason) {
+//		if (reason == null) {
+//			exportButton.setEnabled(true);
+//			exportWarning.setText(null);
+//		}
+//		else {
+//			exportButton.disable(reason);
+//			exportWarning.setText("   " + reason);
+//		}
+//	}
+	
+	private String checkSpeciesManager(PamDataBlock dataBlock) {
+		DataBlockSpeciesManager spManager = dataBlock.getDatablockSpeciesManager();
+		if (spManager == null) {
+			return "No species manager";
+		}
+		String error = spManager.checkSpeciesMapError();
+		return error;
 	}
 
 	public void showPopup(MouseEvent e, int row) {
@@ -259,4 +305,15 @@ public class DatablockSynchPanel extends TethysGUIPanel {
 		}
 		
 	}
+
+	@Override
+	protected void exportButtonPressed(ActionEvent e) {
+		exportData();
+	}
+
+	@Override
+	protected void optionsButtonPressed(ActionEvent e) {
+		// TODO Auto-generated method stub
+		
+	}
 }

src/tethys/swing/DeploymentExportPanel.java

@@ -34,6 +34,7 @@ import tethys.TethysState;
 import tethys.TethysState.StateType;
 import tethys.dbxml.DBXMLConnect;
 import tethys.deployment.DeploymentHandler;
+import tethys.deployment.RecordingList;
 import tethys.deployment.RecordingPeriod;
 import tethys.niluswraps.PDeployment;
 
@@ -241,7 +242,9 @@ public class DeploymentExportPanel extends TethysGUIPanel implements DeploymentT
 		if (selectedDeployments == null || selectedDeployments.size() == 0) {
 			return;
 		};
-		getTethysControl().getDeploymentHandler().exportDeployments(selectedDeployments);
+		// need to turn that list back into a RecordingList object. 
+		RecordingList tempList = new RecordingList("eport list", selectedDeployments);
+		getTethysControl().getDeploymentHandler().exportDeployments(tempList);
 	}
 	
 

src/tethys/swing/DeploymentsPanel.java

@@ -16,81 +16,38 @@ import javax.swing.border.TitledBorder;
 import PamView.panel.PamPanel;
 import tethys.TethysControl;
 import tethys.TethysState;
+import tethys.calibration.CalibrationHandler;
 import tethys.deployment.DeploymentHandler;
+import tethys.deployment.RecordingList;
 import tethys.deployment.RecordingPeriod;
 
-public class DeploymentsPanel extends TethysGUIPanel implements DeploymentTableObserver {
+public class DeploymentsPanel extends TethysExportPanel implements DeploymentTableObserver {
 	
-	private JPanel mainPanel;
+//	private JPanel mainPanel;
 	
 	private PAMGuardDeploymentsTable pamDeploymentsTable;
 	
 	private DeploymentExportPanel exportPanel;
 	
-	private JButton exportButton, optionsButton;
-//	private TethysDeploymentsTable tethysDeploymentsTable;
-	private JLabel exportWarning;
+	private JLabel effortName;
 
 	public DeploymentsPanel(TethysControl tethysControl) {
-		super(tethysControl);
+		super(tethysControl, tethysControl.getDeploymentHandler(), true);
 		DeploymentHandler deploymentHandler = tethysControl.getDeploymentHandler();
 		pamDeploymentsTable = new PAMGuardDeploymentsTable(tethysControl);
 		exportPanel = new DeploymentExportPanel(tethysControl, pamDeploymentsTable);
 		pamDeploymentsTable.addObserver(exportPanel);
-//		tethysDeploymentsTable = new TethysDeploymentsTable(tethysControl);
-		mainPanel = new PamPanel(new BorderLayout());
+		
+		JPanel mainPanel = getMainPanel();
 		mainPanel.setBorder(new TitledBorder("Recording periods and deployment information"));
 		pamDeploymentsTable.addObserver(this);
 		pamDeploymentsTable.addObserver(deploymentHandler);
-//		JSplitPane splitPane = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT);
-//		splitPane.add(pamDeploymentsTable.getComponent());
-//		splitPane.add(tethysDeploymentsTable.getComponent());
-//		mainPanel.add(splitPane,BorderLayout.CENTER);
-//		SwingUtilities.invokeLater(new Runnable() {
-//			
-//			@Override
-//			public void run() {
-//				splitPane.setDividerLocation(0.6);
-//			}
-//		});
-		JPanel ctrlPanel = new PamPanel(new BorderLayout());
-		JPanel ctrlButtons = new JPanel();
-		ctrlButtons.setLayout(new BoxLayout(ctrlButtons, BoxLayout.X_AXIS));
-		optionsButton = new JButton("Options ...");
-		exportButton = new JButton("Export ...");
-		ctrlButtons.add(optionsButton);
-		ctrlButtons.add(exportButton);
-		ctrlPanel.add(BorderLayout.WEST, ctrlButtons);
 		
-		optionsButton.addActionListener(new ActionListener() {
-			@Override
-			public void actionPerformed(ActionEvent e) {
-				getTethysControl().getDeploymentHandler().showOptions(null);
-			}
-		});
-		
-		exportButton.addActionListener(new ActionListener() {
-			@Override
-			public void actionPerformed(ActionEvent e) {
-				exportDeployments();
-			}
-		});
-		exportWarning = new JLabel(" ");
-		ctrlPanel.add(BorderLayout.CENTER, exportWarning);
-		
-		mainPanel.add(BorderLayout.CENTER, pamDeploymentsTable.getComponent());
-		mainPanel.add(BorderLayout.NORTH, ctrlPanel);
-//		mainPanel.add(BorderLayout.EAST, exportPanel.getComponent());
-		exportButton.setEnabled(false);
-	}
-
-	protected void exportDeployments() {
-		getTethysControl().getDeploymentHandler().exportDeployments();
-	}
-
-	@Override
-	public JComponent getComponent() {
-		return mainPanel;
+		effortName = new JLabel("   ");
+		JPanel centralPanel = new JPanel(new BorderLayout());
+		centralPanel.add(BorderLayout.NORTH, effortName);
+		centralPanel.add(BorderLayout.CENTER,pamDeploymentsTable.getComponent());
+		mainPanel.add(BorderLayout.CENTER, centralPanel);
 	}
 
 	@Override
@@ -98,12 +55,21 @@ public class DeploymentsPanel extends TethysGUIPanel implements DeploymentTableO
 		enableExportButton();
 	}
 	
-	
-
 	private void enableExportButton() {
+		if (!getTethysControl().isServerOk()) {
+			disableExport("Tethys server not running");
+			return;
+		}
+		
+		CalibrationHandler calHandler = getTethysControl().getCalibrationHandler();
+		if (calHandler.haveAllChannelCalibrations() == false) {
+			disableExport("Calibration data for each channel must be exported before creating Deployment documents");
+			return;
+		}
+		
 		ArrayList<RecordingPeriod> selected = pamDeploymentsTable.getSelectedPeriods();
-		if (selected == null) {
-			exportButton.setEnabled(false);
+		if (selected == null || selected.size() == 0) {
+			disableExport("You must select one or more deployment periods to export");
 			return;
 		}
 		boolean existing = false;
@@ -118,17 +84,35 @@ public class DeploymentsPanel extends TethysGUIPanel implements DeploymentTableO
 		}
 		String warning = null;
 		if (existing) {
-			warning = "  One or more deployment documents already exist. These must be deleted prior to exporting new documents";
-			exportWarning.setText(warning);
+			warning = "One or more deployment documents already exist. These must be deleted prior to exporting new documents";
+			disableExport(warning);
+			return;
 		}
-
-		exportButton.setEnabled(selected.size()>0 & existing == false && getTethysControl().isServerOk());
+		
+		enableExport(true);
 	}
 
 	@Override
 	public void updateState(TethysState tethysState) {
 		super.updateState(tethysState);
 		enableExportButton();
+		RecordingList recordingList = pamDeploymentsTable.getMasterList();
+		if (recordingList == null) {
+			effortName.setText("  No available effort data");
+		}
+		else {
+			effortName.setText("  Effort from " + recordingList.getSourceName());
+		}
+	}
+
+	@Override
+	protected void exportButtonPressed(ActionEvent e) {
+		getTethysControl().getDeploymentHandler().exportDeployments();
+	}
+
+	@Override
+	protected void optionsButtonPressed(ActionEvent e) {
+		getTethysControl().getDeploymentHandler().showOptions(null);
 	}
 
 

src/tethys/swing/FancyClientButton.java

@@ -1,12 +1,15 @@
 package tethys.swing;
 
 import java.awt.BorderLayout;
+import java.awt.Desktop;
 import java.awt.FlowLayout;
 import java.awt.GridBagConstraints;
 import java.awt.GridBagLayout;
 import java.awt.Insets;
 import java.awt.event.ActionEvent;
 import java.awt.event.ActionListener;
+import java.io.File;
+import java.io.IOException;
 
 import javax.swing.AbstractButton;
 import javax.swing.BoxLayout;
@@ -30,14 +33,14 @@ import tethys.dbxml.DBXMLConnect;
  */
 public class FancyClientButton extends JPanel {
 
+	private TethysControl tethysControl;
+	
 	private JButton clientButton;
 	private JButton dropButton;
 	private JPopupMenu collectionsMenu;
-	private TethysControl tethysControl;
 	private JCheckBoxMenuItem showBrowser;
 	private AbstractButton showPAMGuard;
 
-
 	public FancyClientButton(TethysControl tethysControl) {
 		this.tethysControl = tethysControl;
 		setLayout(new GridBagLayout());
@@ -105,6 +108,16 @@ public class FancyClientButton extends JPanel {
 			menuItem.addActionListener(new OpenCollection(collections[i]));
 			collectionsMenu.add(menuItem);
 		}
+		collectionsMenu.addSeparator();
+		JMenuItem tmpItem = new JMenuItem("Open temp folder");
+		collectionsMenu.add(tmpItem);
+		tmpItem.setToolTipText("Open folder used for temporary document files during export in Windows Explorer");
+		tmpItem.addActionListener(new ActionListener() {
+			@Override
+			public void actionPerformed(ActionEvent e) {
+				openTempFolder();
+			}
+		});
 
 		dropButton.addActionListener(new ActionListener() {
 			@Override
@@ -115,6 +128,19 @@ public class FancyClientButton extends JPanel {
 		enableItems();
 	}
 
+	protected void openTempFolder() {
+		File tempFolder = tethysControl.getDbxmlConnect().checkTempFolder();
+		if (tempFolder == null) {
+			return;
+		}
+		try {
+			Desktop.getDesktop().open(tempFolder);
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+
 	protected void enableItems() {
 		boolean isP = tethysControl.getTethysExportParams().listDocsInPamguard;
 		showBrowser.setSelected(!isP);

src/tethys/swing/PAMGuardDeploymentsTable.java

@@ -32,6 +32,7 @@ import tethys.TethysState.StateType;
 import tethys.dbxml.TethysException;
 import tethys.deployment.DeploymentHandler;
 import tethys.deployment.DeploymentOverview;
+import tethys.deployment.RecordingList;
 import tethys.deployment.RecordingPeriod;
 import tethys.niluswraps.PDeployment;
 
@@ -55,6 +56,8 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 	
 	private ArrayList<DeploymentTableObserver> observers = new ArrayList<>();
 
+private RecordingList masterList;
+
 	public PAMGuardDeploymentsTable(TethysControl tethysControl) {
 		super(tethysControl);
 //		deploymentHandler = new DeploymentHandler(getTethysControl());
@@ -74,6 +77,10 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 	public JComponent getComponent() {
 		return mainPanel;
 	}
+	
+	public RecordingList getMasterList() {
+		return masterList;
+	}
 
 	private class TableMouse extends MouseAdapter {
 
@@ -95,7 +102,7 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 		public void mouseClicked(MouseEvent e) {
 			int aRow = table.getSelectedRow();
 			int col = table.getSelectedColumn();
-			ArrayList<RecordingPeriod> periods = deploymentOverview.getRecordingPeriods();
+			ArrayList<RecordingPeriod> periods = getMasterList().getEffortPeriods();
 			if (aRow >= 0 && aRow < periods.size() && col == TableModel.SELECTCOLUMN) {
 				periods.get(aRow).toggleSelected();
 				notifyObservers();
@@ -118,7 +125,8 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 		}
 		// make a list of RecordingPeriods which don't currently have a Deployment document
 		ArrayList<RecordingPeriod> newPeriods = new ArrayList<>();
-		ArrayList<RecordingPeriod> allPeriods = deploymentOverview.getRecordingPeriods();
+		ArrayList<RecordingPeriod> allPeriods = getMasterList().getEffortPeriods();
+//		ArrayList<RecordingPeriod> allPeriods = deploymentOverview.getRecordingPeriods();
 		ArrayList<PDeployment> matchedDeployments = new ArrayList<>();
 		for (int i = 0; i < selRows.length; i++) {
 			PDeployment tethysDeployment = allPeriods.get(selRows[i]).getMatchedTethysDeployment();
@@ -155,14 +163,6 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 		}
 		
 		if (matchedDeployments.size() == 1) {
-			menuItem = new JMenuItem("Delete deployment document " + matchedDeployments.get(0));
-			menuItem.addActionListener(new ActionListener() {
-				@Override
-				public void actionPerformed(ActionEvent e) {
-					deleteDeployment(matchedDeployments.get(0));
-				}
-			});
-			popMenu.add(menuItem);
 			menuItem = new JMenuItem("Display deployment document " + matchedDeployments.get(0));
 			menuItem.addActionListener(new ActionListener() {
 				@Override
@@ -179,7 +179,17 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 				}
 			});
 			popMenu.add(menuItem);
+
+			popMenu.addSeparator();
 			
+			menuItem = new JMenuItem("Delete deployment document " + matchedDeployments.get(0));
+			menuItem.addActionListener(new ActionListener() {
+				@Override
+				public void actionPerformed(ActionEvent e) {
+					deleteDeployment(matchedDeployments.get(0));
+				}
+			});
+			popMenu.add(menuItem);
 			
 		}
 		else if (matchedDeployments.size() > 1){
@@ -200,7 +210,8 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 	}
 	
 	protected void selectAll(boolean select) {
-		ArrayList<RecordingPeriod> recordingPeriods = deploymentOverview.getRecordingPeriods();
+		ArrayList<RecordingPeriod> recordingPeriods = getMasterList().getEffortPeriods();
+//		ArrayList<RecordingPeriod> recordingPeriods = deploymentOverview.getRecordingPeriods();
 		for (int i = 0; i < recordingPeriods.size(); i++) {
 			recordingPeriods.get(i).setSelected(select);
 		}
@@ -329,7 +340,7 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 		if (deploymentOverview == null) {
 			return null;
 		}
-		ArrayList<RecordingPeriod> allPeriods = deploymentOverview.getRecordingPeriods();
+		ArrayList<RecordingPeriod> allPeriods = getMasterList().getEffortPeriods();
 		ArrayList<RecordingPeriod> selPeriods = new ArrayList();
 		int n = allPeriods.size();
 		for (int i = 0; i < n; i++) {
@@ -348,6 +359,7 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 	private void updateDeployments() {
 		DeploymentHandler deploymentHandler = getTethysControl().getDeploymentHandler();
 		deploymentOverview = deploymentHandler.getDeploymentOverview();
+		masterList = deploymentOverview.getMasterList(getTethysControl());
 		if (deploymentOverview == null) {
 			return;
 		}
@@ -373,7 +385,7 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 				return 0;
 			}
 			else {
-				return deploymentOverview.getRecordingPeriods().size();
+				return getMasterList().size();
 			}
 		}
 
@@ -398,13 +410,14 @@ public class PAMGuardDeploymentsTable extends TethysGUIPanel {
 
 		@Override
 		public Object getValueAt(int rowIndex, int columnIndex) {
-			RecordingPeriod period = deploymentOverview.getRecordingPeriods().get(rowIndex);
+			RecordingList masterList = getMasterList();
+			RecordingPeriod period = masterList.getEffortPeriods().get(rowIndex);
 //			DeploymentRecoveryPair deplInfo = deploymentInfo.get(rowIndex);
 			if (columnIndex == 6) {
-				return deploymentOverview.getDutyCycleInfo();
+				return masterList.assessDutyCycle();
 			}
 			if (columnIndex == 4 && rowIndex > 0) {
-				RecordingPeriod prevPeriod = deploymentOverview.getRecordingPeriods().get(rowIndex-1);
+				RecordingPeriod prevPeriod = masterList.getEffortPeriods().get(rowIndex-1);
 				long gap = period.getRecordStart() - prevPeriod.getRecordStop();
 				return PamCalendar.formatDuration(gap);
 			}

src/tethys/swing/TethysConnectionPanel.java

@@ -277,12 +277,17 @@ public class TethysConnectionPanel extends TethysGUIPanel {
 	@Override
 	public void updateState(TethysState tethysState) {
 		super.updateState(tethysState);
-		if (tethysState.stateType == StateType.UPDATESERVER) {
+		switch (tethysState.stateType) {
+		case UPDATESERVER:
 			fillServerControl();
 			updateProjectList();
-		}
-		if (tethysState.stateType == StateType.NEWPROJECTSELECTION) {
+			break;
+		case NEWPROJECTSELECTION:
 			updateProjectList();
+			break;
+		case UPDATEMETADATA:
+			updateInstrumentsList();
+			break;
 		}
 	}
 

src/tethys/swing/TethysDeploymentsTable.java

@@ -21,6 +21,7 @@ import tethys.TethysControl;
 import tethys.TethysMenuActions;
 import tethys.TethysState;
 import tethys.deployment.DeploymentOverview;
+import tethys.deployment.RecordingList;
 import tethys.niluswraps.PDeployment;
 
 public class TethysDeploymentsTable extends TethysGUIPanel {
@@ -121,8 +122,7 @@ public class TethysDeploymentsTable extends TethysGUIPanel {
 		@Override
 		public String getColumnName(int column) {
 			return columnNames[column];
-		}
-		
+		}		
 
 		public String getMatchText(PDeployment deployment) {
 			// TODO Auto-generated method stub
@@ -132,8 +132,9 @@ public class TethysDeploymentsTable extends TethysGUIPanel {
 			if (deploymentOverview == null) {
 				return "No PAMGuard data";
 			}
-			Long depStart = deploymentOverview.getFirstStart();
-			Long depEnd = deploymentOverview.getLastEnd();
+			RecordingList masterList = deploymentOverview.getMasterList(getTethysControl());
+			Long depStart = masterList.getStart();
+			Long depEnd = masterList.getEnd();
 			if (depStart == null) {
 				return "No PAMGuard recordings";
 			}

src/tethys/swing/TethysExportPanel.java

@@ -0,0 +1,227 @@
+package tethys.swing;
+
+import java.awt.BorderLayout;
+import java.awt.FlowLayout;
+import java.awt.event.ActionEvent;
+import java.awt.event.ActionListener;
+
+import javax.swing.BoxLayout;
+import javax.swing.JButton;
+import javax.swing.JComponent;
+import javax.swing.JLabel;
+import javax.swing.JPanel;
+
+import PamView.dialog.NorthPanel;
+import PamView.dialog.SettingsButton;
+import PamView.help.PamHelp;
+import tethys.Collection;
+import tethys.CollectionHandler;
+import tethys.TethysControl;
+
+/**
+ * Common panel used by Calibrations, Deployments and Detections to show an export button and other 
+ * common components, such as help, a table, etc.. 
+ * @author dg50
+ *
+ */
+abstract public class TethysExportPanel extends TethysGUIPanel {
+	
+	private TippedButton exportButton;
+	
+	private JButton optionsButton, helpButton;
+	
+	private JPanel mainPanel, northPanel;
+	
+	private JLabel message;
+
+	private CollectionHandler collectionHandler;
+
+	private String helpPoint;
+
+	private boolean showOptions;
+
+	public TethysExportPanel(TethysControl tethysControl, CollectionHandler collectionHandler, boolean showOptions) {
+		super(tethysControl);
+		this.collectionHandler = collectionHandler;
+		this.showOptions = showOptions;
+		this.helpPoint = collectionHandler.getHelpPoint();
+		
+		mainPanel = new JPanel(new BorderLayout());
+		northPanel = new JPanel(new BorderLayout());
+		JPanel nwPanel = new JPanel();
+		nwPanel.setLayout(new BoxLayout(nwPanel, BoxLayout.X_AXIS));
+		JPanel nePanel = new JPanel();
+		nePanel.setLayout(new BoxLayout(nePanel, BoxLayout.X_AXIS));
+		northPanel.add(BorderLayout.CENTER, nwPanel);
+		northPanel.add(BorderLayout.EAST, nePanel);
+		mainPanel.add(BorderLayout.NORTH, northPanel);
+		
+		optionsButton = new SettingsButton();
+		exportButton = new TippedButton("Export ...", "Export " + collectionHandler.collectionName() + " to Tethys");
+		helpButton = new JButton("?");
+		helpButton.setToolTipText("Show context sensitive help");
+		JLabel space = new JLabel("  ");
+		message = new JLabel (" ");
+		
+		nwPanel.add(optionsButton);
+		nwPanel.add(exportButton);
+		nwPanel.add(space);
+		nwPanel.add(message);
+		nePanel.add(helpButton);
+		
+		showAndHide();
+		
+		optionsButton.addActionListener(new ActionListener() {
+			@Override
+			public void actionPerformed(ActionEvent e) {
+				optionsButtonPressed(e);
+			}
+		});
+		exportButton.addActionListener(new ActionListener() {
+			@Override
+			public void actionPerformed(ActionEvent e) {
+				exportButtonPressed(e);
+			}
+		});
+		helpButton.addActionListener(new ActionListener() {
+			
+			@Override
+			public void actionPerformed(ActionEvent e) {
+				helpButtonPressed(e);
+			}
+		});
+	}
+
+	/**
+	 * Show the help. 
+	 * @param e
+	 */
+	protected void helpButtonPressed(ActionEvent e) {
+		if (helpPoint == null) {
+			return;
+		}
+		PamHelp.getInstance().displayContextSensitiveHelp(helpPoint);
+	}
+
+	/**
+	 * Export button has been pressed. 
+	 * @param e
+	 */
+	protected abstract void exportButtonPressed(ActionEvent e);
+
+	/**
+	 * Options button has been pressed. 
+	 * @param e
+	 */
+	protected abstract void optionsButtonPressed(ActionEvent e);
+
+	private void showAndHide() {
+		optionsButton.setVisible(showOptions);
+		helpButton.setVisible(helpPoint != null);
+	}
+
+	@Override
+	public JComponent getComponent() {
+		return mainPanel;
+	}
+
+	/**
+	 * @return the helpPoint
+	 */
+	public String getHelpPoint() {
+		return helpPoint;
+	}
+
+	/**
+	 * @param helpPoint the helpPoint to set
+	 */
+	public void setHelpPoint(String helpPoint) {
+		this.helpPoint = helpPoint;
+		showAndHide();
+	}
+
+	/**
+	 * @return the showOptions
+	 */
+	public boolean isShowOptions() {
+		return showOptions;
+	}
+
+	/**
+	 * @param showOptions the showOptions to set
+	 */
+	public void setShowOptions(boolean showOptions) {
+		this.showOptions = showOptions;
+		showAndHide();
+	}
+
+	/**
+	 * @return the exportButton
+	 */
+	public TippedButton getExportButton() {
+		return exportButton;
+	}
+
+	/**
+	 * @return the optionsButton
+	 */
+	public JButton getOptionsButton() {
+		return optionsButton;
+	}
+
+	/**
+	 * @return the helpButton
+	 */
+	public JButton getHelpButton() {
+		return helpButton;
+	}
+
+	/**
+	 * @return the mainPanel
+	 */
+	public JPanel getMainPanel() {
+		return mainPanel;
+	}
+
+	/**
+	 * @return the northPanel
+	 */
+	public JPanel getNorthPanel() {
+		return northPanel;
+	}
+
+	/**
+	 * @return the message
+	 */
+	public JLabel getMessage() {
+		return message;
+	}
+
+	/**
+	 * @return the collectionHandler
+	 */
+	public CollectionHandler getCollectionHandler() {
+		return collectionHandler;
+	}
+	
+	/**
+	 * Enable or disable export button, leaving tool tips alone
+	 * @param enable
+	 */
+	public void enableExport(boolean enable) {
+		exportButton.setEnabled(enable);
+		if (enable) {
+			message.setText(null);
+		}
+	}
+	
+	/**
+	 * Disable the export button and set the tooltip. 
+	 * @param disabledTip
+	 */
+	public void disableExport(String disabledTip) {
+		exportButton.disable(disabledTip);
+		message.setText(disabledTip);
+	}
+
+}

src/tethys/swing/TippedButton.java

@@ -0,0 +1,77 @@
+package tethys.swing;
+
+import javax.swing.JButton;
+
+public class TippedButton extends JButton {
+
+	private static final long serialVersionUID = 1L;
+	private String enabledTip;
+	private String disabledTip;
+
+	/**
+	 * Create a button with standard tips which will be used for enabled state
+	 * @param text
+	 * @param enabledTip
+	 */
+	public TippedButton(String text, String enabledTip) {
+		this(text, enabledTip, null);
+	}
+	
+	/**
+	 * Create a button with standard tips which will be used for enabled and disabled state
+	 * @param text
+	 * @param enabledTip
+	 * @param disabledTip
+	 */
+	public TippedButton(String text, String enabledTip, String disabledTip) {
+		super(text);
+		this.enabledTip = enabledTip;
+		this.disabledTip = disabledTip;
+		setToolTipText(enabledTip);
+	}
+
+	@Override
+	public void setEnabled(boolean enable) {
+		super.setEnabled(enable);
+		setToolTipText(enable ? enabledTip : disabledTip);
+	}
+	
+	/*
+	 * Call to disable the button and at the same time 
+	 * set a tooltip giving the reason. 
+	 */
+	public void disable(String newTip) {
+		disabledTip = newTip;
+		setEnabled(false);
+	}
+
+	/**
+	 * @return the enabledTip
+	 */
+	public String getEnabledTip() {
+		return enabledTip;
+	}
+
+	/**
+	 * @param enabledTip the enabledTip to set
+	 */
+	public void setEnabledTip(String enabledTip) {
+		this.enabledTip = enabledTip;
+	}
+
+	/**
+	 * @return the disabledTip
+	 */
+	public String getDisabledTip() {
+		return disabledTip;
+	}
+
+	/**
+	 * @param disabledTip the disabledTip to set
+	 */
+	public void setDisabledTip(String disabledTip) {
+		this.disabledTip = disabledTip;
+	}
+	
+
+}

src/tethys/swing/XMLStringView.java

@@ -28,6 +28,9 @@ public class XMLStringView extends PamDialog {
 		
 		getCancelButton().setVisible(false);
 		setModal(false);
+		
+		getOkButton().setText("Close");
+		getOkButton().setToolTipText("Close window");
 	}
 	
 	public static void showDialog(Window parent, String collection, String documentId, String xmlString) {
@@ -38,7 +41,7 @@ public class XMLStringView extends PamDialog {
 
 	@Override
 	public boolean getParams() {
-		return false;
+		return true;
 	}
 
 	@Override

src/tethys/swing/export/DeploymentPeriodPanel.java

@@ -19,6 +19,7 @@ import metadata.PamguardMetaData;
 import nilus.Deployment;
 import nilus.DeploymentRecoveryDetails;
 import tethys.TethysTimeFuncs;
+import tethys.tooltips.TethysTips;
 
 public class DeploymentPeriodPanel {
 
@@ -68,6 +69,9 @@ public class DeploymentPeriodPanel {
 				enableControls();
 			}
 		});
+
+		deploymentStart.setToolTipText(TethysTips.findTip(DeploymentRecoveryDetails.class, "TimeStamp"));
+		deploymentEnd.setToolTipText(TethysTips.findTip(DeploymentRecoveryDetails.class, "TimeStamp"));
 	}
 
 	protected void enableControls() {

src/tethys/swing/export/DescriptionTypePanel.java

@@ -19,6 +19,7 @@ import PamView.PamGui;
 import PamView.dialog.PamDialog;
 import nilus.DescriptionType;
 import tethys.niluswraps.WrappedDescriptionType;
+import tethys.tooltips.TethysTips;
 
 /**
  * Panel containing the three test entry fields for nilus.DescriptionType
@@ -73,6 +74,10 @@ public class DescriptionTypePanel {
 		addScrollablePanel(tObjectives, "Objectives");
 		addScrollablePanel(tAbstract, "Abstract");
 		addScrollablePanel(tMethod, "Method");
+		
+		tObjectives.setToolTipText(TethysTips.Detections_Description_Objectives);
+		tAbstract.setToolTipText(TethysTips.Detections_Description_Abstract);
+		tMethod.setToolTipText(TethysTips.Detections_Description_Method);
 	}
 
 	private void addScrollablePanel(JTextArea textArea, String title) {

src/tethys/swing/export/ExportWorkerCard.java

@@ -147,7 +147,7 @@ public class ExportWorkerCard extends ExportWizardCard implements DetectionExpor
 				totExpected *= progress.exportCount/(progress.exportCount+progress.skipCount);
 			}
 			projectedCount.setText(String.format("%d",  totExpected));
-			long perc = (progress.exportCount+progress.skipCount) * 100 / progress.totalUnits;
+			long perc = (progress.doneMapPoints) * 100 / progress.nMapPoints;
 			progressBar.setValue((int) perc);
 		}
 		switch (progress.state) {
@@ -176,6 +176,18 @@ public class ExportWorkerCard extends ExportWizardCard implements DetectionExpor
 		boolean stopped = state == DetectionExportProgress.STATE_CANCELED || state == DetectionExportProgress.STATE_COMPLETE;
 		export.setEnabled(stopped);
 		cancel.setEnabled(!stopped);
+		detectionsExportWizard.getCancelButton().setEnabled(stopped);
+		detectionsExportWizard.getPreviousButton().setEnabled(stopped);
+	}
+
+	@Override
+	public void setVisible(boolean visible) {
+		/**
+		 * setVisible is called by the wizard for each component, so 
+		 * we can see here if we're on this or not. 
+		 */
+		super.setVisible(visible);
+		detectionsExportWizard.getOkButton().setEnabled(!visible);
 	}
 
 }

src/tethys/swing/export/ResponsiblePartyPanel.java

@@ -10,6 +10,7 @@ import javax.swing.border.TitledBorder;
 
 import PamView.dialog.PamGridBagContraints;
 import nilus.ContactInfo.Address;
+import tethys.tooltips.TethysTips;
 import nilus.ResponsibleParty;
 
 /**
@@ -56,6 +57,11 @@ public class ResponsiblePartyPanel {
 		c.gridx = 0;
 		c.gridy++;
 		
+		name.setToolTipText("Person responsible for data");
+		organisation.setToolTipText("Responsible organization");
+		position.setToolTipText("Persons role in organization");
+		email.setToolTipText("email address or other contact details");
+		
 	}
 
 	public JPanel getMainPanel() {

src/tethys/tooltips/TethysTips.java

@@ -0,0 +1,403 @@
+package tethys.tooltips;
+
+import java.lang.reflect.Field;
+
+import nilus.Deployment;
+
+/**
+ * Class to make it easy to find tooltips for a given nilus class and field nams. The constatns
+ * could be used directly, or can be found using findTip(Class, String) using the class type
+ * and field name for any nilus object. 
+ * Tips were generates from a set of csv files extracted from the xml schema using Matlab code, then formatted
+ * in Matlab and pasted into this class, so will need to rerun that process should the xml schemas be updated. 
+ * 
+ * @author dg50
+ *
+ */
+public class TethysTips {
+	
+//	public static void main(String[] args) {
+//		Class cls = Deployment.Data.class;
+//		String field = "Audio";
+//		String foundTip = findTip(cls, field);
+//		System.out.println(foundTip);
+//	}
+	
+	/**
+	 * find the tooltip for a given class and field within that class. 
+	 * @param aClass
+	 * @param field
+	 * @return found tip or null
+	 */
+	public static String findTip(Class aClass, String fieldName) {
+		Package pack = aClass.getPackage();
+		String packName = pack.toString();
+		String clsName = aClass.getCanonicalName();
+		if (clsName.startsWith("nilus.") == false) {
+			return null;
+		}
+		clsName = clsName.substring(6);
+		clsName = clsName.replace('.', '_');
+		String varName = clsName + "_" + fieldName;
+		// now try to find that field in this class and get it's value. 
+		Field field = null;
+		try {
+			field = TethysTips.class.getDeclaredField(varName);
+		} catch (NoSuchFieldException | SecurityException e) {
+			return null;
+		}
+		if (field == null) {
+			return null;
+		}
+		Object tip = null;
+		try {
+			tip = field.get(null);
+		} catch (IllegalArgumentException | IllegalAccessException e) {
+			return null;
+		}
+		if (tip == null) {
+			return null;
+		}
+		
+		return tip.toString();
+	}
+
+	// Annotations taken from schemata_csv
+	public static final String Calibration_Id = "Identifier of instrument, preamplifier, or hydrophone. Corresponds to elements in Deployment: Deployment/Instrument/Id, Deployment/Sensors/Audio/HydrophoneId, Deployment/Sensors/Audio[i]/PreampId. As instruments may be calibrated multiple times, it is not an error for duplicate Id values to appear. It is recommended that the three different types of identifiers (instrument, hydrophone, preamp) be distinct, but the Type element may be used to distinguish them if they are not.";
+	public static final String Calibration_TimeStamp = "Date and time of calibration";
+	public static final String Calibration_Type = "hydrophone, preamplifier, or end-to-end Indicates type of calibration";
+	public static final String Calibration_Process = "Process used to calibrate instrument.";
+	public static final String Calibration_ResponsibleParty = "Who conducted/managed the calibration?";
+	public static final String Calibration_IntensityReference_uPa = "Reference intensity in µ Pascals for dB measurements. Commonly used: underwater acoustics: 1 terrestrial acoustics: 20";
+	public static final String Calibration_Sensitivity_dBV = "Optional measurement of transducer sensitivity at 1 kHz.";
+	public static final String Calibration_Sensitivity_V = "Optional measurement of Sensitivity_dBV on a linear scale, provided by many transducer manufacturers.";
+	public static final String Calibration_Sensitivity_dBFS = "Optional measurement for digital transducers. Digital transducers do not output voltage measurements. In this case, the 1 kHz sensitivity measurement is measured relative to peak output of the full-scale signal instead of RMS. It should be noted that for sinusoidal signals, the RMS sensitivity will be 3 dB lower (Lewis, 2012). Lewis, J. (2012). \"Understanding Microphone Sensitivity,\" Analog Dialogue 46(2). 14-16.";
+	public static final String Calibration_FrequencyResponse = "Lists of frequencies (Hz) and responses (dB). Lists must be of equal length.";
+	public static final String Calibration_MetadataInfo = "Information about who is responsible for this metadata record, when it was created, and how often it is updated.";
+	public static final String Calibration_Process_Method = "Text based description of algorithm or citation";
+	public static final String Calibration_Process_Software = "Name of software that implements the algorithm or supports human analysts. This might be the name of a plug-in or extension module that is part of a larger program or system.";
+	public static final String Calibration_Process_Version = "Software version identifier";
+	public static final String Calibration_Process_Parameters = "Structured tags to describe parameters used in algorithm execution.";
+	public static final String Calibration_Process_SupportSoftware = "Software required in addition to the algorithm itself, e.g. Matlab, Ishmael, PAMGUARD, Triton, etc.";
+	public static final String Calibration_Process_SupportSoftware_Version = "Software version identifier.";
+	public static final String Calibration_Process_SupportSoftware_Parameters = "Structured tags to describe parameters used in algorithm execution.";
+	public static final String Calibration_ResponsibleParty_contactInfo_onlineResource = "We do not fully conform to the onlineResources of ISO 19115";
+	public static final String Calibration_QualityAssurance_Quality = "Measurement is: unverified, valid, invalid";
+	public static final String Calibration_QualityAssurance_AlternateCalibration = "Provide an alternative calibration Id that should be used (if available) when the Quality value is invalid.";
+	public static final String Calibration_MetadataInfo_Contact = "based on ISO 19115";
+	public static final String Calibration_MetadataInfo_Date = "Last update.";
+	public static final String Calibration_MetadataInfo_UpdateFrequency = "How often are these data updated? as-needed, unplanned, or yearly";
+	public static final String Calibration_MetadataInfo_Contact_contactInfo_onlineResource = "We do not fully conform to the onlineResources of ISO 19115";
+
+	// Annotations taken from schemata_csv
+	public static final String Deployment_Id = "Character sequence that uniquely identifies this deployment.";
+	public static final String Deployment_Description = "Objectives, abstract and high-level methods.";
+	public static final String Deployment_Project = "Name of project associated with this deployment. Can be related to a geographic region, funding source, etc.";
+	public static final String Deployment_DeploymentId = "A number related to either the Nth deployment operation in a series of deployments or the Nth deployment at a specific site. This is different from Id which is a unqiue identifier for the deployment. If a vessel deployed 5 instruments, they might all have the same DeploymentId. While not enforced, it is expected that the combination of Project, DeploymentId, and (Site or Cruise) to be unique.";
+	public static final String Deployment_DeploymentAlias = "Alternative deployment description.";
+	public static final String Deployment_Site = "Name of a location where instruments are frequently deployed. Can be something as simple as a letter or a geographic name. Strongly recommended for long-term time series recorded at a specific point.";
+	public static final String Deployment_SiteAliases = "Alternative names for the deployment location";
+	public static final String Deployment_Cruise = "Name of deployment cruise.";
+	public static final String Deployment_Platform = "On what platform is the instrument deployed? (e.g. mooring, tag)";
+	public static final String Deployment_Region = "Name of geographic region.";
+	public static final String Deployment_Instrument = "Instrument type and identifier.";
+	public static final String Deployment_SamplingDetails = "Information about recordings on each channel. Sample rate, quantization bits, etc.";
+	public static final String Deployment_Data = "Data from instrument, a URI is provided when not present (typical for audio).";
+	public static final String Deployment_DeploymentDetails = "Instrument deployment location, time, etc.";
+	public static final String Deployment_RecoveryDetails = "Instrument recovery, location, time, etc.";
+	public static final String Deployment_Sensors = "Sensors on instrument.";
+	public static final String Deployment_MetadataInfo = "Party responsible for this record. Some applications may make this mandatory.";
+	public static final String Deployment_Description_Objectives = "What are the objectives of this effort? Examples: Beamform to increase SNR for detection. Detect every click of a rare species. Verify data quality.";
+	public static final String Deployment_Description_Abstract = "Overview of effort.";
+	public static final String Deployment_Description_Method = "High-level description of the method used.";
+	public static final String Deployment_Instrument_Type = "Instrument type, e.g. HARP, EAR, Popup, DMON, Rock Hopper, etc.";
+	public static final String Deployment_Instrument_InstrumentId = "Instrument identifier, e.g. serial number";
+	public static final String Deployment_Instrument_GeometryType = "Sensor attachment \"rigid\" - relative geometry is fixed, \"cabled\" - relative geometry may be expected to deform depending on movement, currents, etc.";
+	public static final String Deployment_SamplingDetails_Channel_ChannelNumber = "Channels and sensors are bound together from Start to End. While not enforced, we assume channels are numbered from 1 to N.";
+	public static final String Deployment_SamplingDetails_Channel_SensorNumber = "Audio sensor index within the Sensors element. This allows us to associate a channel with a physical hydrophone.";
+	public static final String Deployment_SamplingDetails_Channel_Sampling = "Sampling rate and quantization may change over time.";
+	public static final String Deployment_SamplingDetails_Channel_Gain = "Initial gain setting (assumed 0 if not populated) and any subsequent changes.";
+	public static final String Deployment_SamplingDetails_Channel_DutyCycle = "Duty cycle is represented by the recording duration and the interval from the start of one recording session to the next. A duration of 3 m and an interval of 5 m would represent a 60% duty cycle, 3 m on, 2 m off.";
+	public static final String Deployment_SamplingDetails_Channel_Sampling_Regimen = "Sampling regimen may change over time. Each entry shows the start of a sampling configuration.";
+	public static final String Deployment_SamplingDetails_Channel_Sampling_Regimen_SampleRate_kHz = "Number of samples per second in kHz, e.g. 192 is 192,000 samples/s";
+	public static final String Deployment_SamplingDetails_Channel_Sampling_Regimen_SampleBits = "Number of bits per sample.";
+	public static final String Deployment_SamplingDetails_Channel_Gain_Regimen_Gain_rel = "Only used if gain is not calibrated. Relative gain may be a number on a dial.";
+	public static final String Deployment_SamplingDetails_Channel_DutyCycle_Regimen = "Duty cycling regimen may change over time. Each entry shows the start of a duty cycle configuration. The abscence of entries indicates continuous sampling as would having equal values in RecordingDuration_m and RecordingInterval_m.";
+	public static final String Deployment_SamplingDetails_Channel_DutyCycle_Regimen_TimeStamp = "Indicates when the duty cycle becomes active. It remains active until the next Regimen entry.";
+	public static final String Deployment_SamplingDetails_Channel_DutyCycle_Regimen_RecordingDuration_s = "The amount of time in minutes during each recording interval when the data logger is recoring. Use the attribute Offset_s when the recording does not begin at the beginning of each recording interval.";
+	public static final String Deployment_SamplingDetails_Channel_DutyCycle_Regimen_RecordingInterval_s = "Time between consecutive recordings. If RecordingDuration_s is 1800 s and RecordingInterval_s is 3600 s, then we record for the 30 min of each hour.";
+	public static final String Deployment_QualityAssurance_Description = "Text based description of process.";
+	public static final String Deployment_QualityAssurance_ResponsibleParty = "based on ISO 19115";
+	public static final String Deployment_QualityAssurance_Quality = "If no quality assurance, create an entry of Category unverified spanning the acoustic record.";
+	public static final String Deployment_QualityAssurance_Description_Objectives = "What are the objectives of this effort? Examples: Beamform to increase SNR for detection. Detect every click of a rare species. Verify data quality.";
+	public static final String Deployment_QualityAssurance_Description_Abstract = "Overview of effort.";
+	public static final String Deployment_QualityAssurance_Description_Method = "High-level description of the method used.";
+	public static final String Deployment_QualityAssurance_ResponsibleParty_contactInfo_onlineResource = "We do not fully conform to the onlineResources of ISO 19115";
+	public static final String Deployment_QualityAssurance_Quality_Category = "categories: unverified, good, compromised, unusable";
+	public static final String Deployment_QualityAssurance_Quality_FrequencyRange = "QA metric applies to what frequency range?";
+	public static final String Deployment_QualityAssurance_Quality_Comment = "Additional qualitative information";
+	public static final String Deployment_Data_Audio = "Information about audio data.";
+	public static final String Deployment_Data_Tracks = "A set of measurements about a ship/instrument's track line.";
+	public static final String Deployment_Data_Audio_URI = "Uniform Resource Indicator that points to audio content. Examples: digital object identifier, web address, or even a simple string describing the storage location.";
+	public static final String Deployment_Data_Audio_Processed = "Pointer to location of data that has been processed (e.g. checked for quality, decimated, etc.)";
+	public static final String Deployment_Data_Audio_Raw = "Pointer to raw data from the instrument.";
+	public static final String Deployment_Data_Tracks_Track = "A set of sorted (by time) points associated with one or more tracklines.";
+	public static final String Deployment_Data_Tracks_TrackEffort = "Not all measurements are associated with an instrument/ship's planned trackline (e.g. when in chase mode or transiting between tracklines). Specify times for track effort here if needed.";
+	public static final String Deployment_Data_Tracks_URI = "Pointer to trackline information.";
+	public static final String Deployment_Data_Tracks_Track_TrackId = "Optional trackline number. If unimportant, everything can be put in one Points element.";
+	public static final String Deployment_Data_Tracks_Track_Point = "Timestamped measurements: long/lat, bearing, etc. Points should be sorted by timestamp.";
+	public static final String Deployment_Data_Tracks_Track_Point_Bearing_DegN = "Bearing in degrees [0, 360) relative to true or magnetic north (as specified by north attribute, default magnetic)";
+	public static final String Deployment_Data_Tracks_Track_Point_Speed_kn = "Speed in knots";
+	public static final String Deployment_Data_Tracks_Track_Point_Pitch_deg = "Instrument pitch [0, 360) degrees";
+	public static final String Deployment_Data_Tracks_Track_Point_Roll_deg = "Instrument roll [0, 360) degrees";
+	public static final String Deployment_Data_Tracks_Track_Point_Elevation_m = "Instrument elevation (meters) relative to average sea level.";
+	public static final String Deployment_Data_Tracks_Track_Point_GroundElevation_m = "Ground or seabed elevation (meters) relative to average sea level.";
+	public static final String Deployment_Data_Tracks_Track_Point_Longitude = "Expressed in degrees East [0, 360)";
+	public static final String Deployment_Data_Tracks_Track_Point_Latitude = "Expressed in degrees North [-90, 90]";
+	public static final String Deployment_Data_Tracks_TrackEffort_OnPath_FocalArea = "This element is used to provide names that specify a focal area in which the study was conducted, such as a National Marine Sanctuary.";
+	public static final String Deployment_Data_Tracks_TrackEffort_OffPath_FocalArea = "This element is used to provide names that specify a focal area in which the study was conducted, such as a National Marine Sanctuary.";
+	public static final String Deployment_DeploymentDetails_Longitude = "Expressed in degrees East [0, 360)";
+	public static final String Deployment_DeploymentDetails_Latitude = "Expressed in degrees North [-90, 90]";
+	public static final String Deployment_DeploymentDetails_ElevationInstrument_m = "The elevation at which this instrument is positioned.";
+	public static final String Deployment_DeploymentDetails_DepthInstrument_m = "Not usually required. This field is designed to record depth with respect to the ground or seabed. Uses for this field include mines and alpine lakes.";
+	public static final String Deployment_DeploymentDetails_Elevation_m = "Elevation of ground/sea bed";
+	public static final String Deployment_DeploymentDetails_TimeStamp = "Time at which instrument was deployed/recovered. Lost instruments: set recovery time to deployment time.";
+	public static final String Deployment_DeploymentDetails_AudioTimeStamp = "Recording start or end - May differ from deployment time.";
+	public static final String Deployment_DeploymentDetails_ResponsibleParty = "based on ISO 19115";
+	public static final String Deployment_DeploymentDetails_ResponsibleParty_contactInfo_onlineResource = "We do not fully conform to the onlineResources of ISO 19115";
+	public static final String Deployment_RecoveryDetails_Longitude = "Expressed in degrees East [0, 360)";
+	public static final String Deployment_RecoveryDetails_Latitude = "Expressed in degrees North [-90, 90]";
+	public static final String Deployment_RecoveryDetails_ElevationInstrument_m = "The elevation at which this instrument is positioned.";
+	public static final String Deployment_RecoveryDetails_DepthInstrument_m = "Not usually required. This field is designed to record depth with respect to the ground or seabed. Uses for this field include mines and alpine lakes.";
+	public static final String Deployment_RecoveryDetails_Elevation_m = "Elevation of ground/sea bed";
+	public static final String Deployment_RecoveryDetails_TimeStamp = "Time at which instrument was deployed/recovered. Lost instruments: set recovery time to deployment time.";
+	public static final String Deployment_RecoveryDetails_AudioTimeStamp = "Recording start or end - May differ from deployment time.";
+	public static final String Deployment_RecoveryDetails_ResponsibleParty = "based on ISO 19115";
+	public static final String Deployment_RecoveryDetails_ResponsibleParty_contactInfo_onlineResource = "We do not fully conform to the onlineResources of ISO 19115";
+	public static final String Deployment_Sensors_Audio_Number = "Sensor index. May be used to associate the sensor with other parts of the schema. For example, for Audio sensors, the Channel/SensorNumber can be set to a specific Sensor/Audio/Number, permitting us to determine information about the a hydrophone assembly.";
+	public static final String Deployment_Sensors_Audio_SensorId = "A value that uniquely identifies this sensor, e.g. a serial number.";
+	public static final String Deployment_Sensors_Audio_Geometry = "Geometry relative to platform";
+	public static final String Deployment_Sensors_Audio_Name = "Optional sensor name";
+	public static final String Deployment_Sensors_Audio_Description = "Optional description of sensor.";
+	public static final String Deployment_Sensors_Audio_HydrophoneId = "Optional hydrophone identifier.";
+	public static final String Deployment_Sensors_Audio_PreampId = "Optional preamplifier identifier.";
+	public static final String Deployment_Sensors_Depth_Number = "Sensor index. May be used to associate the sensor with other parts of the schema. For example, for Audio sensors, the Channel/SensorNumber can be set to a specific Sensor/Audio/Number, permitting us to determine information about the a hydrophone assembly.";
+	public static final String Deployment_Sensors_Depth_SensorId = "A value that uniquely identifies this sensor, e.g. a serial number.";
+	public static final String Deployment_Sensors_Depth_Geometry = "Geometry relative to platform";
+	public static final String Deployment_Sensors_Depth_Name = "Optional sensor name";
+	public static final String Deployment_Sensors_Depth_Description = "Optional description of sensor.";
+	public static final String Deployment_Sensors_Sensor_Number = "Sensor index. May be used to associate the sensor with other parts of the schema. For example, for Audio sensors, the Channel/SensorNumber can be set to a specific Sensor/Audio/Number, permitting us to determine information about the a hydrophone assembly.";
+	public static final String Deployment_Sensors_Sensor_SensorId = "A value that uniquely identifies this sensor, e.g. a serial number.";
+	public static final String Deployment_Sensors_Sensor_Geometry = "Geometry relative to platform";
+	public static final String Deployment_Sensors_Sensor_Name = "Optional sensor name";
+	public static final String Deployment_Sensors_Sensor_Description = "Optional description of sensor.";
+	public static final String Deployment_Sensors_Sensor_Type = "Description of data gathered by this sensor, e.g., temperature";
+	public static final String Deployment_Sensors_Sensor_Properties = "List of property elements describing the sensor. These may be arbitrary. Example: Properties can have child Units with value °C. Children may be nested.";
+	public static final String Deployment_MetadataInfo_Contact = "based on ISO 19115";
+	public static final String Deployment_MetadataInfo_Date = "Last update.";
+	public static final String Deployment_MetadataInfo_UpdateFrequency = "How often are these data updated? as-needed, unplanned, or yearly";
+	public static final String Deployment_MetadataInfo_Contact_contactInfo_onlineResource = "We do not fully conform to the onlineResources of ISO 19115";
+
+
+	// Annotations taken from schemata_csv
+	public static final String Detections_Id = "Identification string that is unique to all documents of this type (currently optional, will be required in the future)";
+	public static final String Detections_Description = "Objectives, abstract and high-level methods.";
+	public static final String Detections_DataSource = "Acoustic data identifier.";
+	public static final String Detections_Algorithm = "Detailed methods.";
+	public static final String Detections_QualityAssurance = "Description of quality assurance checks (if any).";
+	public static final String Detections_UserId = "User that submitted the document.";
+	public static final String Detections_Effort = "Span and scope of detection effort.";
+	public static final String Detections_OnEffort = "Collection of individual detections.";
+	public static final String Detections_OffEffort = "Collection of off-effort (ad-hoc) detections. Each detection has the same format as the OnEffort ones.";
+	public static final String Detections_MetadataInfo = "Party responsible for this record. Some applications may make this mandatory.";
+	public static final String Detections_Description_Objectives = "What are the objectives of this effort? Examples: Beamform to increase SNR for detection. Detect every click of a rare species. Verify data quality.";
+	public static final String Detections_Description_Abstract = "Overview of effort.";
+	public static final String Detections_Description_Method = "High-level description of the method used.";
+	public static final String Detections_DataSource_EnsembleId = "Serves as a foreign key into the ensembles collection and must match an Id element in an ensemble document. Ensembles are used to group instruments together for a common purpose (e.g. large aperture array).";
+	public static final String Detections_DataSource_DeploymentId = "Serves as a foreign key into the Deployments collection and must match an Id element in a deployment document.";
+	public static final String Detections_Algorithm_Method = "Text based description of algorithm or citation";
+	public static final String Detections_Algorithm_Software = "Name of software that implements the algorithm or supports human analysts. This might be the name of a plug-in or extension module that is part of a larger program or system.";
+	public static final String Detections_Algorithm_Version = "Software version identifier";
+	public static final String Detections_Algorithm_Parameters = "Structured tags to describe parameters used in algorithm execution.";
+	public static final String Detections_Algorithm_SupportSoftware = "Software required in addition to the algorithm itself, e.g. Matlab, Ishmael, PAMGUARD, Triton, etc.";
+	public static final String Detections_Algorithm_SupportSoftware_Version = "Software version identifier.";
+	public static final String Detections_Algorithm_SupportSoftware_Parameters = "Structured tags to describe parameters used in algorithm execution.";
+	public static final String Detections_QualityAssurance_Description = "Text based description of process.";
+	public static final String Detections_QualityAssurance_ResponsibleParty = "based on ISO 19115";
+	public static final String Detections_QualityAssurance_Description_Objectives = "What are the objectives of this effort? Examples: Beamform to increase SNR for detection. Detect every click of a rare species. Verify data quality.";
+	public static final String Detections_QualityAssurance_Description_Abstract = "Overview of effort.";
+	public static final String Detections_QualityAssurance_Description_Method = "High-level description of the method used.";
+	public static final String Detections_QualityAssurance_ResponsibleParty_contactInfo_onlineResource = "We do not fully conform to the onlineResources of ISO 19115";
+	public static final String Detections_Effort_Start = "Timestamp indicating the start of systematic effort to find the species and phenomena listed in the Effort/Kind entries.";
+	public static final String Detections_Effort_End = "Timestamp indicating end of systematic effort.";
+	public static final String Detections_Effort_dBReferenceIntensity_uPa = "All dB measurements are made relative to this value in uPa. Typical values are 1 for underwater acoustics and 20 for terrestrial acoustics.";
+	public static final String Detections_Effort_AnalysisGaps_Aperiodic = "Used to describe meaningful gaps in the analysis effort. Problems with the data should not be addressed here, but rather in Deployment/QualityAssurance/Quality. Note that tools may not take Gaps into account when reporting effort statistics.";
+	public static final String Detections_Effort_AnalysisGaps_Periodic_Regimen = "Peridoic analysis regimen may change over time. Each entry shows the start of an analysis regimen. The abscence of entries indicates continuous analysis as would having equal values in AnalysisDuration_s and AnalysisInterval_s. The time offsets in these fields are with respect to actual time. Duty cycled data are not taken into account in their specification. As an example, if we analyzed the first 30 min of each hour and the deployment's recording duty cycle were 15 min of recording every 30 min, this analysis duration would only result in 15 min of analysis every hour.";
+	public static final String Detections_Effort_AnalysisGaps_Periodic_Regimen_TimeStamp = "Indicates when the regimen becomes active. It remains active until the next Regimen entry.";
+	public static final String Detections_Effort_AnalysisGaps_Periodic_Regimen_AnalysisDuration_s = "When analysis starts, the data are analyzed for this many seconds. Optional attribute Offset_s may be used to denote the number of seconds after the timestamp that analysis started. If Offset_s is not present, analysis starts at the Timestamp.";
+	public static final String Detections_Effort_AnalysisGaps_Periodic_Regimen_AnalysisInterval_s = "Time between consecutive effort. If AnalysisDuration_s is 1800 s and AnalysisInterval_s is 3600 s, then we perform analysis on the first 30 min of each hour starting at TimeStamp.";
+	public static final String Detections_Effort_AnalysisGaps_Aperiodic_Start = "Timestamp indicating the start of a gap in the systematic effort to find the species and phenomena listed in the Effort/Kind entries.";
+	public static final String Detections_Effort_AnalysisGaps_Aperiodic_End = "Timestamp indicating end of systematic effort gap.";
+	public static final String Detections_Effort_AnalysisGaps_Aperiodic_Reason = "Reason for gap in analysis.";
+	public static final String Detections_Effort_Kind_SpeciesId = "Integrated Taxonomic Information System species identifier http://www.itis.gov/ for positive numbers. Negative numbers are used for physical phenomena.";
+	public static final String Detections_Effort_Kind_Call = "Name that describes call.";
+	public static final String Detections_Effort_Kind_Granularity = "Type of detections: call - individual call, encounter - set of calls, binned - presence detected within period specified by bin size attribute in Effort. grouped – A set of individual detections of any granularity that have been grouped together. Examples include situations such as song or other groupings (e.g. detections of the same animals picked up on multiple instruments). Grouped detections may specify the individual detections regardless of their granularity that are part of the group. This is different from granularities encounter and binned where one might expect multiple calls to occur, but the individual detections are not recorded.";
+	public static final String Detections_Effort_Kind_Parameters_Subtype = "subcategory of call";
+	public static final String Detections_Effort_Kind_Parameters_FrequencyMeasurements_Hz = "Specifies a list of frequencies at which measurements are made. Each detection for this Kind should have a list of FrequencyMeasurements where each item corresponds to a frequency in this list. Useful for studying ambient sound or soundscapes. Be sure to declare Effort/ReferenceIntensity_uPa.";
+	public static final String Detections_OnEffort_Detection_Input_file = "Optional name of audio file (or indirect representation) from which this detection was generated.";
+	public static final String Detections_OnEffort_Detection_Start = "Time at which event started. For many detectors, this may not the actual starting time of the event.";
+	public static final String Detections_OnEffort_Detection_End = "Optional end time of event.";
+	public static final String Detections_OnEffort_Detection_Count = "An optional count of the number of times a call occurred within a bin or across an encounter.";
+	public static final String Detections_OnEffort_Detection_Event = "Optional tag for identifying this event uniquely within the stream. For human analysts, it is typical to use the time at which the detection was made in ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ). When present, the combination of the event and attributes that uniquely identify the set of detections (or document name) must be uniqe.";
+	public static final String Detections_OnEffort_Detection_UnitId = "Specifies ensemble unit (when using an ensemble source).";
+	public static final String Detections_OnEffort_Detection_SpeciesId = "Integrated Taxonomic Information System species identifier http://www.itis.gov/ for positive numbers. Negative numbers are used for physical phenomena.";
+	public static final String Detections_OnEffort_Detection_Call = "In most cases, the call field should be present. May be omitted if the goal is species detection only, or repeated for multiple types of calls when the granularity effort is not \"call\".";
+	public static final String Detections_OnEffort_Detection_Image = "Name of image file (spectrogram, etc.)";
+	public static final String Detections_OnEffort_Detection_Audio = "Name of audio file (short snippet)";
+	public static final String Detections_OnEffort_Detection_Parameters_Subtype = "subcategory of call";
+	public static final String Detections_OnEffort_Detection_Parameters_Score = "Measure from detector, e.g. likelihood ratio, projection, etc.";
+	public static final String Detections_OnEffort_Detection_Parameters_Confidence = "Measure of confidence in detection. Range: [0, 1]";
+	public static final String Detections_OnEffort_Detection_Parameters_QualityAssurance = "Detection is: unverified, valid, invalid";
+	public static final String Detections_OnEffort_Detection_Parameters_ReceivedLevel_dB = "dB relative to reference intensity defined in Effort/ReferenceIntenstiy_uPa";
+	public static final String Detections_OnEffort_Detection_Parameters_FrequencyMeasurements_dB = "List of received levels at various frequencies relative to the reference value defiend in Effort/ReferenceIntensity_uPa. The frequency measurements should be consistent for each species and call type and must correspond to a a list of frequencies defined in Effort/Kind/SubType.";
+	public static final String Detections_OnEffort_Detection_Parameters_Peaks_Hz = "Typically used for spectra of short echolocation bursts, notes the spectral peaks in a list sorted from low to high frequency.";
+	public static final String Detections_OnEffort_Detection_Parameters_Duration_s = "When the call granularity is binned or encounter, this may be used to describe the mean duration of calls during the bout. As an example, at SIO we use this to track the mean duration of binned anthropogenic sources such as explosions.";
+	public static final String Detections_OnEffort_Detection_Parameters_Sideband_Hz = "Signal sideband frequencies in a list sorted from low to high frequency.";
+	public static final String Detections_OnEffort_Detection_Parameters_EventRef = "Reference to other detections for hierarchical organization.";
+	public static final String Detections_OnEffort_Detection_Parameters_UserDefined = "Study specific parameters";
+	public static final String Detections_OnEffort_Detection_Parameters_Tonal_Offset_s = "List of offsets from start in seconds";
+	public static final String Detections_OnEffort_Detection_Parameters_Tonal_Hz = "Frequency measurement for each Offset_s (Hz). List must be of same length as Offset_s";
+	public static final String Detections_OnEffort_Detection_Parameters_Tonal_dB = "Optional intensity measurment (dB) for each Offset_s (dB). List must be of the same length as Offset_s";
+	public static final String Detections_OffEffort_Detection_Input_file = "Optional name of audio file (or indirect representation) from which this detection was generated.";
+	public static final String Detections_OffEffort_Detection_Start = "Time at which event started. For many detectors, this may not the actual starting time of the event.";
+	public static final String Detections_OffEffort_Detection_End = "Optional end time of event.";
+	public static final String Detections_OffEffort_Detection_Count = "An optional count of the number of times a call occurred within a bin or across an encounter.";
+	public static final String Detections_OffEffort_Detection_Event = "Optional tag for identifying this event uniquely within the stream. For human analysts, it is typical to use the time at which the detection was made in ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ). When present, the combination of the event and attributes that uniquely identify the set of detections (or document name) must be uniqe.";
+	public static final String Detections_OffEffort_Detection_UnitId = "Specifies ensemble unit (when using an ensemble source).";
+	public static final String Detections_OffEffort_Detection_SpeciesId = "Integrated Taxonomic Information System species identifier http://www.itis.gov/ for positive numbers. Negative numbers are used for physical phenomena.";
+	public static final String Detections_OffEffort_Detection_Call = "In most cases, the call field should be present. May be omitted if the goal is species detection only, or repeated for multiple types of calls when the granularity effort is not \"call\".";
+	public static final String Detections_OffEffort_Detection_Image = "Name of image file (spectrogram, etc.)";
+	public static final String Detections_OffEffort_Detection_Audio = "Name of audio file (short snippet)";
+	public static final String Detections_OffEffort_Detection_Parameters_Subtype = "subcategory of call";
+	public static final String Detections_OffEffort_Detection_Parameters_Score = "Measure from detector, e.g. likelihood ratio, projection, etc.";
+	public static final String Detections_OffEffort_Detection_Parameters_Confidence = "Measure of confidence in detection. Range: [0, 1]";
+	public static final String Detections_OffEffort_Detection_Parameters_QualityAssurance = "Detection is: unverified, valid, invalid";
+	public static final String Detections_OffEffort_Detection_Parameters_ReceivedLevel_dB = "dB relative to reference intensity defined in Effort/ReferenceIntenstiy_uPa";
+	public static final String Detections_OffEffort_Detection_Parameters_FrequencyMeasurements_dB = "List of received levels at various frequencies relative to the reference value defiend in Effort/ReferenceIntensity_uPa. The frequency measurements should be consistent for each species and call type and must correspond to a a list of frequencies defined in Effort/Kind/SubType.";
+	public static final String Detections_OffEffort_Detection_Parameters_Peaks_Hz = "Typically used for spectra of short echolocation bursts, notes the spectral peaks in a list sorted from low to high frequency.";
+	public static final String Detections_OffEffort_Detection_Parameters_Duration_s = "When the call granularity is binned or encounter, this may be used to describe the mean duration of calls during the bout. As an example, at SIO we use this to track the mean duration of binned anthropogenic sources such as explosions.";
+	public static final String Detections_OffEffort_Detection_Parameters_Sideband_Hz = "Signal sideband frequencies in a list sorted from low to high frequency.";
+	public static final String Detections_OffEffort_Detection_Parameters_EventRef = "Reference to other detections for hierarchical organization.";
+	public static final String Detections_OffEffort_Detection_Parameters_UserDefined = "Study specific parameters";
+	public static final String Detections_OffEffort_Detection_Parameters_Tonal_Offset_s = "List of offsets from start in seconds";
+	public static final String Detections_OffEffort_Detection_Parameters_Tonal_Hz = "Frequency measurement for each Offset_s (Hz). List must be of same length as Offset_s";
+	public static final String Detections_OffEffort_Detection_Parameters_Tonal_dB = "Optional intensity measurment (dB) for each Offset_s (dB). List must be of the same length as Offset_s";
+	public static final String Detections_MetadataInfo_Contact = "based on ISO 19115";
+	public static final String Detections_MetadataInfo_Date = "Last update.";
+	public static final String Detections_MetadataInfo_UpdateFrequency = "How often are these data updated? as-needed, unplanned, or yearly";
+	public static final String Detections_MetadataInfo_Contact_contactInfo_onlineResource = "We do not fully conform to the onlineResources of ISO 19115";
+
+
+	// Annotations taken from schemata_csv
+	public static final String Ensemble_Id = "Ensemble name (unique identifier).";
+	public static final String Ensemble_Unit = "Associates a virtual unit of an ensemble with an actual deployment.";
+	public static final String Ensemble_ZeroPosition = "Provides a zero point to which relative localizations can be referenced.";
+	public static final String Ensemble_Unit_UnitId = "A unique unit number that identifies this instrument within the ensemble.";
+	public static final String Ensemble_Unit_DeploymentId = "Reference to a deployment document Id field. Uniquely identifies the deployment.";
+	public static final String Ensemble_ZeroPosition_Longitude = "Expressed in degrees East [0, 360)";
+	public static final String Ensemble_ZeroPosition_Latitude = "Expressed in degrees North [-90, 90]";
+
+
+	// Annotations taken from schemata_csv
+	public static final String Localize_Id = "Identification string that is unique to all documents of this type";
+	public static final String Localize_Description = "Text based description of process.";
+	public static final String Localize_DataSource = "Indicates the deployment or ensemble from which the process (e.g. detector) derived information.";
+	public static final String Localize_Algorithm = "Description of an algorithm or process.";
+	public static final String Localize_QualityAssurance = "Description of quality assurance checks (if any).";
+	public static final String Localize_ResponsibleParty = "Person/organization responsible for generating metadata";
+	public static final String Localize_UserId = "User that submitted the document.";
+	public static final String Localize_IntermediateData = "Derived data that is used for the localizations that the user wishes to retain.";
+	public static final String Localize_MetadataInfo = "Party responsible for this record. Some applications may make this mandatory.";
+	public static final String Localize_Description_Objectives = "What are the objectives of this effort? Examples: Beamform to increase SNR for detection. Detect every click of a rare species. Verify data quality.";
+	public static final String Localize_Description_Abstract = "Overview of effort.";
+	public static final String Localize_Description_Method = "High-level description of the method used.";
+	public static final String Localize_DataSource_EnsembleId = "Serves as a foreign key into the ensembles collection and must match an Id element in an ensemble document. Ensembles are used to group instruments together for a common purpose (e.g. large aperture array).";
+	public static final String Localize_DataSource_DeploymentId = "Serves as a foreign key into the Deployments collection and must match an Id element in a deployment document.";
+	public static final String Localize_Algorithm_Method = "Text based description of algorithm or citation";
+	public static final String Localize_Algorithm_Software = "Name of software that implements the algorithm or supports human analysts. This might be the name of a plug-in or extension module that is part of a larger program or system.";
+	public static final String Localize_Algorithm_Version = "Software version identifier";
+	public static final String Localize_Algorithm_Parameters = "Structured tags to describe parameters used in algorithm execution.";
+	public static final String Localize_Algorithm_SupportSoftware = "Software required in addition to the algorithm itself, e.g. Matlab, Ishmael, PAMGUARD, Triton, etc.";
+	public static final String Localize_Algorithm_SupportSoftware_Version = "Software version identifier.";
+	public static final String Localize_Algorithm_SupportSoftware_Parameters = "Structured tags to describe parameters used in algorithm execution.";
+	public static final String Localize_QualityAssurance_Description = "Text based description of process.";
+	public static final String Localize_QualityAssurance_ResponsibleParty = "based on ISO 19115";
+	public static final String Localize_QualityAssurance_Description_Objectives = "What are the objectives of this effort? Examples: Beamform to increase SNR for detection. Detect every click of a rare species. Verify data quality.";
+	public static final String Localize_QualityAssurance_Description_Abstract = "Overview of effort.";
+	public static final String Localize_QualityAssurance_Description_Method = "High-level description of the method used.";
+	public static final String Localize_QualityAssurance_ResponsibleParty_contactInfo_onlineResource = "We do not fully conform to the onlineResources of ISO 19115";
+	public static final String Localize_ResponsibleParty_contactInfo_onlineResource = "We do not fully conform to the onlineResources of ISO 19115";
+	public static final String Localize_Effort_Start = "Time at which we started looking for location information.";
+	public static final String Localize_Effort_End = "Time at which we stopped looking for location information.";
+	public static final String Localize_Effort_CoordinateSystem = "What type of localization information is produced?";
+	public static final String Localize_Effort_LocalizationType = "Type of localization effort: Bearing, Ranging, Point, Track";
+	public static final String Localize_Effort_CoordinateSystem_Type = "How are positions represented? WGS84: global positioning system lat/long, cartesian: m relative to a point, UTM: universal trans Mercatur map-projection, Bearing: Polar measurements of angle and possibly distance.";
+	public static final String Localize_Effort_CoordinateSystem_Relative = "For bearings, this gives the direction vector for the zero bearing. Angles are measured counter-clockwise to this vector. For cartesian coordinates, this provides the zero point relative to the deployment position or the ReferencePoint of an ensemble.";
+	public static final String Localize_Effort_CoordinateSystem_UTM = "Parameters for Universal Trans Mercatur projections. NEED TO INVESTIGATE FURTHER AS TO WHETHER THIS OR ANY OTHER PROJECTION IS WORTH ADDING";
+	public static final String Localize_Effort_CoordinateSystem_Relative_Bearing = "Designed to be a subset of OpenGML DirectionVectorType: http://schemas.opengis.net/gml/3.1.1/base/direction.xsd Unlike the OpenGML, direction may not be specified as a vector, and the verticalAngle is optional.";
+	public static final String Localize_Effort_CoordinateSystem_Relative_Bearing_HorizontalAngle = "Angle between a reference vector in the horizontal plane [0, 360]";
+	public static final String Localize_Effort_CoordinateSystem_Relative_Bearing_VerticalAngle = "Angle between a reference vector in the vertical plane: [-90, 90]";
+	public static final String Localize_Effort_CoordinateSystem_UTM_Zone = "NS zone [1-60]. Each zone covers 80°S to 84°N in 6° width zones. Zone 1 180 is 180-186° E with increasing zone #s corresponding to 6° eastward increments.";
+	public static final String Localize_Effort_ReferencedDocuments_Document_Type = "What type of document is being referenced? Detections or Localizations";
+	public static final String Localize_Effort_ReferencedDocuments_Document_Id = "Unique identifier string for the document being referenced.";
+	public static final String Localize_Effort_ReferencedDocuments_Document_Index = "All localizations that wish to reference other detections or localizations from the referenced document should use this index value.";
+	public static final String Localize_IntermediateData_Correlations_Correlation_Primary = "Primary hydropphone";
+	public static final String Localize_IntermediateData_Correlations_Correlation_Secondary = "Secondary hydropphone";
+	public static final String Localize_IntermediateData_Correlations_Correlation_Correlations = "Correlation between detections on primary hydrophone and signals on secondary hydrophones. Each column j is the set of lags corresponding to the j'th detection on the primary hydrophone.";
+	public static final String Localize_Localizations_Localization_Event = "Optional tag typically in ISO datetime format YYYY-MM-DDTHH:MM:SSZ identifying this event uniquely within the stream. For human analysts, it is typical to use the time at which the detection was made. When present, the combination of the event and attributes that uniquely identify the set of detections (or document name) must be uniqe.";
+	public static final String Localize_Localizations_Localization_TimeStamp = "Time of localization in reference time frame (e.g. time of arrival at primary hydrophone)";
+	public static final String Localize_Localizations_Localization_SpeciesId = "Species can be identified by the detections from the detections that are referenced. As these references are not mandatory, the optional SpeciesID can be used to identify the species that produced the localized source.";
+	public static final String Localize_Localizations_Localization_QualityAssurance = "Detection is: unverified, valid, invalid";
+	public static final String Localize_Localizations_Localization_Bearing = "Direction towards acoustic source.";
+	public static final String Localize_Localizations_Localization_Ranging = "Range and direction towards acoustic source. Combine with bearing, rename angular Rename StdError to Error specify in methods/algorrithm";
+	public static final String Localize_Localizations_Localization_WGS84 = "Longitude, latitude and possibly elevation of surce.";
+	public static final String Localize_Localizations_Localization_Cartesian = "Relative distance to source from receiver zero point.";
+	public static final String Localize_Localizations_Localization_Track = "Series of associated positions and timestamps";
+	public static final String Localize_Localizations_Localization_References_TimeReferenceEnsembleUnit = "STILL NEEDED? Time references which unit of the ensemble (see TimeReferenceChannel) when ensembles are used.";
+	public static final String Localize_Localizations_Localization_References_TimeReferenceChannel = "STILL NEEDED? Events are detected at different times on different channels, making it necessary to provide the instrument and channel on which the timestamp references.";
+	public static final String Localize_Localizations_Localization_References_Reference = "Detections/localization used in constructing this localization.";
+	public static final String Localize_Localizations_Localization_References_Reference_Index = "Must match instance of Index in ReferencedDocuments. This permits identification of a specific document.";
+	public static final String Localize_Localizations_Localization_References_Reference_EventRef = "Event identifier that uniquely identifies a detection or localization within a referenced document.";
+	public static final String Localize_Localizations_Localization_Bearing_HorizontalAngle = "Angle between a reference vector in the horizontal plane [0, 360]";
+	public static final String Localize_Localizations_Localization_Bearing_VerticalAngle = "Angle between a reference vector in the vertical plane: [-90, 90]";
+	public static final String Localize_Localizations_Localization_Bearing_Ambiguous = "Left right horizontal ambiguity about the bearing reference vector exists?";
+	public static final String Localize_Localizations_Localization_Bearing_StdError = "Standard error in degrees for the measurement.";
+	public static final String Localize_Localizations_Localization_Bearing_StdError_HorizontalAngle = "Angle between a reference vector in the horizontal plane [0, 360]";
+	public static final String Localize_Localizations_Localization_Bearing_StdError_VerticalAngle = "Angle between a reference vector in the vertical plane: [-90, 90]";
+	public static final String Localize_Localizations_Localization_Ranging_HorizontalAngle = "Angle between a reference vector in the horizontal plane [0, 360]";
+	public static final String Localize_Localizations_Localization_Ranging_VerticalAngle = "Angle between a reference vector in the vertical plane: [-90, 90]";
+	public static final String Localize_Localizations_Localization_Ranging_Range_m = "Distance to localized animal/object/phenomenon in meters.";
+	public static final String Localize_Localizations_Localization_Ranging_Ambiguous = "Left right horizontal ambiguity about the bearing reference vector exists?";
+	public static final String Localize_Localizations_Localization_Ranging_StdError_HorizontalAngle = "Angle between a reference vector in the horizontal plane [0, 360]";
+	public static final String Localize_Localizations_Localization_Ranging_StdError_VerticalAngle = "Angle between a reference vector in the vertical plane: [-90, 90]";
+	public static final String Localize_Localizations_Localization_Ranging_StdError_Range_m = "Distance to localized animal/object/phenomenon in meters.";
+	public static final String Localize_Localizations_Localization_WGS84_Longitude = "Expressed in degrees East [0, 360)";
+	public static final String Localize_Localizations_Localization_WGS84_Latitude = "Expressed in degrees North [-90, 90]";
+	public static final String Localize_Localizations_Localization_WGS84_AlternatePosition = "Add LongLat3 and StdError Cartesian";
+	public static final String Localize_Localizations_Localization_WGS84_StdError_Longitude = "Expressed in degrees East [0, 360)";
+	public static final String Localize_Localizations_Localization_WGS84_StdError_Latitude = "Expressed in degrees North [-90, 90]";
+	public static final String Localize_Localizations_Localization_Cartesian_BearingIDs = "If multiple bearings were used to create this localization, their ids can be provided.";
+	public static final String Localize_Localizations_Localization_Cartesian_Longitude = "Expressed in degrees East [0, 360)";
+	public static final String Localize_Localizations_Localization_Cartesian_Latitude = "Expressed in degrees North [-90, 90]";
+	public static final String Localize_Localizations_Localization_Cartesian_BearingIDs_EventRef = "Reference to individual bearing within this XML document.";
+	public static final String Localize_Localizations_Localization_Track_WGS84 = "Series of points or list of values for each type long/lat/depth/elevation/timestamp parameters for each call: receivedLevel_dB sourceLevel_dB ambient_dB";
+	public static final String Localize_Localizations_Localization_Track_Cartesian = "Todo: define Cartesian list";
+	public static final String Localize_Localizations_Localization_Track_WGS84_Bounds = "Bounding box for tempo-spatial data from northwest to southeast quadrant. If elevation/depth information is available, separate depth boundaries are given as well. Note that longitudes are always degrees east. When a track crosses 0° east, the northwest longitude will be > than the souteast longitude (e.g. a 2° path from 359°E to 1°E)";
+	public static final String Localize_Localizations_Localization_Track_WGS84_Bounds_NorthWest_Longitude = "Longitude is expressed in degrees East [0,360)";
+	public static final String Localize_Localizations_Localization_Track_WGS84_Bounds_NorthWest_Latitude = "Expressed in degrees North [-90,90]";
+	public static final String Localize_Localizations_Localization_Track_WGS84_Bounds_SouthEast_Longitude = "Longitude is expressed in degrees East [0,360)";
+	public static final String Localize_Localizations_Localization_Track_WGS84_Bounds_SouthEast_Latitude = "Expressed in degrees North [-90,90]";
+	public static final String Localize_MetadataInfo_Contact = "based on ISO 19115";
+	public static final String Localize_MetadataInfo_Date = "Last update.";
+	public static final String Localize_MetadataInfo_UpdateFrequency = "How often are these data updated? as-needed, unplanned, or yearly";
+	public static final String Localize_MetadataInfo_Contact_contactInfo_onlineResource = "We do not fully conform to the onlineResources of ISO 19115";}