DBXM Export

A few functions to help find data.

.classpath

@@ -6,7 +6,7 @@
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/jdk-16.0.2">
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER">
 		<attributes>
 			<attribute name="module" value="true"/>
 			<attribute name="maven.pomderived" value="true"/>

src/PamController/PamguardVersionInfo.java

@@ -24,7 +24,7 @@ public class PamguardVersionInfo {
 	 * PAMGuard can work with. 
 	 */
 	static public final String minJavaVersion = "11.0.0";
-	static public final String maxJavaVersion = "18.99.99";
+	static public final String maxJavaVersion = "19.99.99";
 	
 
 	/**

src/PamguardMVC/PamDataBlock.java

@@ -50,6 +50,8 @@ import org.w3c.dom.Element;
 import Acquisition.AcquisitionControl;
 import Acquisition.AcquisitionProcess;
 import pamScrollSystem.ViewLoadObserver;
+import tethys.pamdata.AutoTethysProvider;
+import tethys.pamdata.TethysDataProvider;
 import dataGram.DatagramProvider;
 import dataMap.BespokeDataMapGraphic;
 import dataMap.OfflineDataMap;
@@ -64,6 +66,7 @@ import PamController.PamController;
 import PamController.PamControllerInterface;
 import PamDetection.LocContents;
 import PamDetection.LocalisationInfo;
+import PamDetection.PamDetection;
 import PamUtils.PamCalendar;
 import PamUtils.PamUtils;
 import PamView.symbol.PamSymbolManager;
@@ -2835,8 +2838,11 @@ public class PamDataBlock<Tunit extends PamDataUnit> extends PamObservable {
 	private Vector<OfflineDataMap> offlineDataMaps = null;
 
 	private SQLLogging logging;
+	
+	private TethysDataProvider tethysDataProvider;
 
 	private JSONObjectDataSource jsonDataSource;
+	
 
 	public Vector<ProcessAnnotation> getProcessAnnotations() {
 		return processAannotations;
@@ -3046,6 +3052,26 @@ public class PamDataBlock<Tunit extends PamDataUnit> extends PamObservable {
 	public SQLLogging getLogging() {
 		return logging;
 	}
+	
+	/**
+	 * Gets a data provider for Tethys. These will probably need
+	 * to be bespoke, but for now will autogenerate based on the SALLogging information. 
+	 * @return the tethysDataProvider
+	 */
+	public TethysDataProvider getTethysDataProvider() {
+		if (tethysDataProvider == null && PamDetection.class.isAssignableFrom(unitClass) && getLogging() != null) {
+			tethysDataProvider = new AutoTethysProvider(this);
+		}
+		return tethysDataProvider;
+	}
+
+	/**
+	 * Set a data provider for Tethys.
+	 * @param tethysDataProvider the tethysDataProvider to set
+	 */
+	public void setTethysDataProvider(TethysDataProvider tethysDataProvider) {
+		this.tethysDataProvider = tethysDataProvider;
+	}
 
 	final public boolean getCanLog() {
 		return (logging != null);

src/clickDetector/layoutFX/clickClassifiers/SweepClassifierSetPaneFX.java

@@ -1600,22 +1600,22 @@ public class SweepClassifierSetPaneFX extends SettingsPane<ClickTypeProperty> {
 			p.setLayout(new GridBagLayout());
 			GridBagConstraints c = new PamGridBagContraints();
 			
-			c.gridx = 0;
-			addComponent(p, enableBearings, c);
-			c.gridx += c.gridwidth;
-			addComponent(p, bearingsExcludeBox, c);
-			c.gridx += c.gridwidth;
-			addComponent(p, new JLabel("bearings between ", JLabel.RIGHT), c);
-			c.gridx += c.gridwidth;
-			addComponent(p, minBearing, c);
-			c.gridx += c.gridwidth;
-			addComponent(p, new JLabel(" and ", JLabel.RIGHT), c);
-			c.gridx += c.gridwidth;
-			addComponent(p, maxBearing, c);
-			c.gridx += c.gridwidth;
-			addComponent(p, new JLabel("(\u00B0)", JLabel.LEFT), c);
-
-			add(BorderLayout.WEST, p);
+//			c.gridx = 0;
+//			addComponent(p, enableBearings, c);
+//			c.gridx += c.gridwidth;
+//			addComponent(p, bearingsExcludeBox, c);
+//			c.gridx += c.gridwidth;
+//			addComponent(p, new JLabel("bearings between ", JLabel.RIGHT), c);
+//			c.gridx += c.gridwidth;
+//			addComponent(p, minBearing, c);
+//			c.gridx += c.gridwidth;
+//			addComponent(p, new JLabel(" and ", JLabel.RIGHT), c);
+//			c.gridx += c.gridwidth;
+//			addComponent(p, maxBearing, c);
+//			c.gridx += c.gridwidth;
+//			addComponent(p, new JLabel("(\u00B0)", JLabel.LEFT), c);
+//
+//			add(BorderLayout.WEST, p);
 			
 			this.multiChan = checkMultiChan();
 
@@ -1661,21 +1661,21 @@ public class SweepClassifierSetPaneFX extends SettingsPane<ClickTypeProperty> {
 			return true;
 		}
 
-		@Override
-		protected void enableControls() {
-				checkMultiChan();
-			
-				enableBearings.setEnabled(multiChan);
-				bearingsExcludeBox.setEnabled(multiChan);
-				minBearing.setEnabled(multiChan);
-				maxBearing.setEnabled(multiChan);
-			
-				if (!multiChan) return;
-				
-				bearingsExcludeBox.setEnabled(enableBearings.isSelected());
-				minBearing.setEnabled(enableBearings.isSelected());
-				maxBearing.setEnabled(enableBearings.isSelected());
-		}
+//		@Override
+//		protected void enableControls() {
+//				checkMultiChan();
+//			
+//				enableBearings.setEnabled(multiChan);
+//				bearingsExcludeBox.setEnabled(multiChan);
+//				minBearing.setEnabled(multiChan);
+//				maxBearing.setEnabled(multiChan);
+//			
+//				if (!multiChan) return;
+//				
+//				bearingsExcludeBox.setEnabled(enableBearings.isSelected());
+//				minBearing.setEnabled(enableBearings.isSelected());
+//				maxBearing.setEnabled(enableBearings.isSelected());
+//		}
 
 		@Override
 		protected void disbleControls(boolean disable) {
@@ -1692,23 +1692,23 @@ public class SweepClassifierSetPaneFX extends SettingsPane<ClickTypeProperty> {
 	private boolean checkMultiChan() {
 		boolean multiChan = false; 
 		//do we have multi-channel clicks?
-		if  (clickControl!=null) {
-			int[] chanGroups = clickControl.getClickParameters().getGroupedSourceParameters().getChannelGroups();
-			multiChan = false;
-			
-			if (chanGroups==null) return multiChan; 
-			
-			for (int i=0; i<chanGroups.length; i++) {
-				int chans = clickControl.getClickParameters().getGroupedSourceParameters().getGroupChannels(i);
-//				Debug.out.println("Check multi-channel: " + chanGroups[i] + "  num: " + PamUtils.getNumChannels(chans));
-				if (PamUtils.getNumChannels(chans)>1) {
-					multiChan = true;
-					break; 
-				}
-			}
-		}
-		else multiChan = true;
-//		Debug.out.println("Check multi-channel: " + multiChan);
+//		if  (clickControl!=null) {
+//			int[] chanGroups = clickControl.getClickParameters().getGroupedSourceParameters().getChannelGroups();
+//			multiChan = false;
+//			
+//			if (chanGroups==null) return multiChan; 
+//			
+//			for (int i=0; i<chanGroups.length; i++) {
+//				int chans = clickControl.getClickParameters().getGroupedSourceParameters().getGroupChannels(i);
+////				Debug.out.println("Check multi-channel: " + chanGroups[i] + "  num: " + PamUtils.getNumChannels(chans));
+//				if (PamUtils.getNumChannels(chans)>1) {
+//					multiChan = true;
+//					break; 
+//				}
+//			}
+//		}
+//		else multiChan = true;
+////		Debug.out.println("Check multi-channel: " + multiChan);
 		
 		return multiChan; 
 	}

src/metadata/MetaDataContol.java

@@ -27,4 +27,18 @@ public class MetaDataContol extends PamControlledUnit {
 		return deploymentSetManager.getMenuItem(parentFrame);
 	}
 
+	/**
+	 * @return the deploymentData
+	 */
+	public DeploymentData getDeploymentData() {
+		return deploymentData;
+	}
+
+	/**
+	 * @param deploymentData the deploymentData to set
+	 */
+	public void setDeploymentData(DeploymentData deploymentData) {
+		this.deploymentData = deploymentData;
+	}
+
 }

src/tethys/TethysControl.java

@@ -15,6 +15,7 @@ import nilus.Deployment;
 import nilus.Deployment.Instrument;
 import tethys.output.StreamExportParams;
 import tethys.output.TethysExportParams;
+import tethys.output.TethysExporter;
 import tethys.output.swing.TethysExportDialog;
 
 /**
@@ -93,5 +94,13 @@ public class TethysControl extends PamControlledUnit {
 		TethysExporter tethysExporter = new TethysExporter(this, tethysExportParams);
 		tethysExporter.doExport();		
 	}
+	
+	/**
+	 * A name for any deta selectors. 
+	 * @return
+	 */
+	public String getDataSelectName() {
+		return getUnitName();
+	}
 
 }

src/tethys/TethysExporter.java

@@ -1,133 +0,0 @@
-package tethys;
-
-import java.util.ArrayList;
-
-import org.w3c.dom.Document;
-
-import PamController.PamControlledUnit;
-import PamController.PamController;
-import PamController.settings.output.xml.PamguardXMLWriter;
-import PamguardMVC.PamDataBlock;
-import generalDatabase.DBSchemaWriter;
-import generalDatabase.SQLLogging;
-import tethys.output.StreamExportParams;
-import tethys.output.TethysExportParams;
-
-public class TethysExporter {
-
-	private TethysControl tethysControl;
-	private TethysExportParams tethysExportParams;
-
-	public TethysExporter(TethysControl tethysControl, TethysExportParams tethysExportParams) {
-		this.tethysControl = tethysControl;
-		this.tethysExportParams = tethysExportParams;
-	}
-	
-	/**
-	 * Does the work. In reality this will need an awful lot of changing, for instance 
-	 * to provide feedback to an observer class to show progress on the display. 
-	 * @return OK if success. 
-	 */
-	public boolean doExport() {
-		/*
-		 * Call some general export function
-		 */
-		exportGeneralData(tethysExportParams);
-		/*
-		 * go through the export params and call something for every 
-		 * data block that's enabled.
-		 */
-		ArrayList<PamDataBlock> allDataBlocks = PamController.getInstance().getDataBlocks();
-		for (PamDataBlock aDataBlock : allDataBlocks) {
-			StreamExportParams streamExportParams = tethysExportParams.getStreamParams(aDataBlock);
-			if (streamExportParams == null || streamExportParams.selected == false) {
-				continue; // not interested in this one. 
-			}
-			exportDataStream(aDataBlock, tethysExportParams, streamExportParams);
-		}
-		/*
-		 * Then do whatever else is needed to complete the document. 
-		 */
-		
-		return true;
-	}
-
-	/**
-	 * No idea if we need this or not. May want to return something different to void, e.g. 
-	 * a reference to the main object for a tethys export. I've no idea !
-	 * @param tethysExportParams2
-	 */
-	private void exportGeneralData(TethysExportParams tethysExportParams) {
-		// TODO Auto-generated method stub
-		
-	}
-
-	/**
-	 * Here is where we export data for a specific data stream to Tethys. 
-	 * 
-	 * @param aDataBlock
-	 * @param tethysExportParams
-	 * @param streamExportParams
-	 */
-	private void exportDataStream(PamDataBlock aDataBlock, TethysExportParams tethysExportParams,
-			StreamExportParams streamExportParams) {
-		/**
-		 * This will probably need to be passed additional parameters and may also want to return something
-		 * other than void in order to build a bigger Tethys document. 
-		 */
-		/*
-		 * Some examples of how to do whatever is needed to get schema and data out of PAMGuard. 
-		 */
-		/*
-		 * first we'll probably want a reference to the module containing the data.
-		 * in principle this can't get null, since the datablock was found be searching in 
-		 * the other direction. 
-		 */
-		PamControlledUnit pamControlledUnit = aDataBlock.getParentProcess().getPamControlledUnit();
-		/*
-		 * Get the XML settings for that datablock. 
-		 */
-		PamguardXMLWriter pamXMLWriter = PamguardXMLWriter.getXMLWriter();
-		Document doc = pamXMLWriter.writeOneModule(pamXMLWriter, System.currentTimeMillis());
-		String moduleXML = null;
-		if (doc != null) {
-			// this string should be XML of all the settings for the module controlling this datablock. 
-			moduleXML = pamXMLWriter.getAsString(doc, true); // change to false to get smaller xml
-			System.out.printf("Module settings for datablock %s are:\n", moduleXML);
-			System.out.println(moduleXML);
-		}
-		/*
-		 * This also should never be null, because we only selected datablocks that had a database
-		 * interface. 
-		 * Future versions may need to change this to use binary stores. This will require 
-		 * the overriding datablock to return something different to SQLLogging - probably a TethysLogging 
-		 * interface, which can probably by default just wrap the SQLLogging , but does allow the 
-		 * option of modifying behaviour and of making something work for binary stores. 
-		 */
-		SQLLogging logging = aDataBlock.getLogging();
-		if (logging == null) return;
-		/**
-		 * From the logging, it's possible to automatically generate a XML schema. This may not
-		 * be entirely right, but will be easy to fix. 
-		 */
-		DBSchemaWriter schemaWriter = new DBSchemaWriter();
-		Document schemaDoc = schemaWriter.generateDatabaseSchema(aDataBlock, logging, logging.getTableDefinition());
-		String schemaXML = null;
-		if (schemaDoc != null) {
-			schemaXML = pamXMLWriter.getAsString(schemaDoc, true);
-		}
-		System.out.printf("Database schema for Module Type %s Name %s are:\n", pamControlledUnit.getUnitType(), pamControlledUnit.getUnitName());
-		System.out.println(schemaXML);
-		
-		/**
-		 * Now can go through the data. Probably, we'll want to go through all the data in 
-		 * the project, but we can hold off on that for now and just go for data that 
-		 * are in memory. We'll also have to think a lot about updating parts of the 
-		 * database which have been reprocessed - what we want to do, should eventually all
-		 * be options set in the dialog and available within TethysExportParams
-		 * For now though, we're just going to export data that are in memory. 
-		 */
-		
-	}
-
-}

src/tethys/dbxml/DBXMLConnect.java

@@ -0,0 +1,24 @@
+package tethys.dbxml;
+
+import tethys.TethysControl;
+
+public class DBXMLConnect {
+
+	private TethysControl tethysControl;
+
+	public DBXMLConnect(TethysControl tethysControl) {
+		this.tethysControl = tethysControl;
+	}
+	
+	public boolean openDatabase() {
+		
+		return true;
+	}
+	
+	public void closeDatabase() {
+		
+	}
+	
+	// add whatever calls are necessary to set up schema's etc. 
+
+}

src/tethys/exchange/TethysExchange.java

@@ -1,5 +0,0 @@
-package tethys.exchange;
-
-public interface TethysExchange {
-//	JAXB
-}

src/tethys/output/TethysExporter.java

@@ -0,0 +1,244 @@
+package tethys.output;
+
+import java.util.ArrayList;
+
+import org.w3c.dom.Document;
+
+import Acquisition.AcquisitionControl;
+import Acquisition.AcquisitionProcess;
+import Array.ArrayManager;
+import Array.Hydrophone;
+import Array.PamArray;
+import Array.SnapshotGeometry;
+import PamController.PamControlledUnit;
+import PamController.PamController;
+import PamController.settings.output.xml.PamguardXMLWriter;
+import PamUtils.PamCalendar;
+import PamguardMVC.PamDataBlock;
+import PamguardMVC.PamDataUnit;
+import generalDatabase.DBSchemaWriter;
+import generalDatabase.SQLLogging;
+import metadata.MetaDataContol;
+import metadata.deployment.DeploymentData;
+import tethys.TethysControl;
+import tethys.dbxml.DBXMLConnect;
+import tethys.pamdata.TethysDataProvider;
+import tethys.pamdata.TethysSchema;
+
+/**
+ * Class sitting at the centre of all operations. It will talk to PAMGuard objects to get schemas and data
+ * and talk to the database connection to move data out (and possibly in). Eventually, a lot of the functionality
+ * in here will be moved to worker threads (SwingWorker?) so that it's easy to keep dialogs alive, show 
+ * progress for big export jobs, etc. For now though, it's a relatively simple set of function which 
+ * we can use to a) open the database, b) check everything such as schemas, etc. c) export data and d) clean up. 
+ * @author dg50
+ *
+ */
+public class TethysExporter {
+
+	private TethysControl tethysControl;
+	private TethysExportParams tethysExportParams;
+	
+	private DBXMLConnect dbxmlConnect;
+
+	public TethysExporter(TethysControl tethysControl, TethysExportParams tethysExportParams) {
+		this.tethysControl = tethysControl;
+		this.tethysExportParams = tethysExportParams;
+		dbxmlConnect = new DBXMLConnect(tethysControl);
+	}
+	
+	/**
+	 * Does the work. In reality this will need an awful lot of changing, for instance 
+	 * to provide feedback to an observer class to show progress on the display. 
+	 * @return OK if success. 
+	 */
+	public boolean doExport() {
+		
+		boolean dbOK = dbxmlConnect.openDatabase();
+		if (!dbOK) {
+			/*
+			 *  should we set up some exceptions to throw ? Can be a lot
+			 *  more informative than a simple 'false'
+			 */
+			return false;
+		}
+		
+		SnapshotGeometry arrayGeometry = findArrayGeometrey();
+		
+		/*
+		 * Call some general export function
+		 */
+		exportGeneralData(tethysExportParams);
+		/*
+		 * go through the export params and call something for every 
+		 * data block that's enabled.
+		 */
+		ArrayList<PamDataBlock> allDataBlocks = PamController.getInstance().getDataBlocks();
+		for (PamDataBlock aDataBlock : allDataBlocks) {
+			StreamExportParams streamExportParams = tethysExportParams.getStreamParams(aDataBlock);
+			if (streamExportParams == null || streamExportParams.selected == false) {
+				continue; // not interested in this one. 
+			}
+			exportDataStream(aDataBlock, tethysExportParams, streamExportParams);
+		}
+		/*
+		 * Then do whatever else is needed to complete the document. 
+		 */
+		
+		dbxmlConnect.closeDatabase();
+		
+		return true;
+	}
+	
+	/**
+	 * find Deployment data. This is stored in a separate PAMGuard module, which may not 
+	 * be present. 
+	 * @return 
+	 */
+	public DeploymentData findDeploymentData() {
+		/**
+		 * What to do if this isn't present or is incomplete ? Should we be showing this in the 
+		 * main export dialog at some point ? More a Q for when we make a nicer UI later in the project. 
+		 */
+		MetaDataContol metaControl = (MetaDataContol) PamController.getInstance().findControlledUnit(MetaDataContol.unitType);
+		if (metaControl == null) {
+			return null;
+		}
+		else {
+			return metaControl.getDeploymentData();
+		}
+	}
+	
+	public SnapshotGeometry findArrayGeometrey() {
+		/*
+		 *  this should never be null, but again, we might want to put some warnings
+		 *  and exception handlers in here anyway. Really just an example to show how
+		 *  to find this. We'll need to dig a bit elsewhere to get more detailed 
+		 *  hydrophone information. 
+		 */
+		/*
+		 * In PAMGuard hydrophones are assigned to streamers, which can have different methods for estimating
+		 * their positions from GPS. The geometry is a sum of xyz in the streamer and xyz in the  hydrophone object
+		 * Within a streamer, hydrophones are considered rigid relative to each other. The stremer will floow 
+		 * a choice of modesl (rigid, threading, etc) to estimate it's position relative to the GPS track. Different 
+		 * errors are used when estimating localisation errors within and between streamers. 
+		 * The Snapshot geometry sorts a lot of this out for a point in time and will give back a single object
+		 * which is most of what we'll be wanting. 
+		 */
+		PamArray currentArray = ArrayManager.getArrayManager().getCurrentArray();
+		SnapshotGeometry currentGeometry = currentArray.getSnapshotGeometry(PamCalendar.getTimeInMillis());
+		/*
+		 * The following lines of code show how to get more detailed calibration info for each 
+		 * hydrophone, but we'll have to think about the easiest way to repackage this for Tethys. 
+		 * e.g. this function could be modified to return the correct Tethys object in one go. 
+		 */
+		ArrayList<Hydrophone> hydrophones = currentArray.getHydrophoneArray();
+		/*
+		 *  each object in the list will have more detailed cal information for each phone. But 
+		 *  for the full system calibration we'd need to go to the Acquisition module.  
+		 */
+		AcquisitionControl daqControl = (AcquisitionControl) PamController.getInstance().findControlledUnit(AcquisitionControl.unitType);
+		if (daqControl != null) {
+			AcquisitionProcess daqProcess = daqControl.getAcquisitionProcess();
+			for (int iPhone = 0; iPhone < hydrophones.size(); iPhone++) {
+			   Hydrophone aPhone = hydrophones.get(iPhone);
+				double totalCal = -daqProcess.rawAmplitude2dB(1, iPhone, false);
+				System.out.printf("hydrophone %d has sensitivity %3.1fdB + gain %3.1fdB. Total calibration is %3.1fdB re1U/uPa\n", 
+						iPhone, aPhone.getSensitivity(), aPhone.getPreampGain(), totalCal);
+			}
+		}
+		
+		
+		return currentGeometry;
+	}
+
+	/**
+	 * No idea if we need this or not. May want to return something different to void, e.g. 
+	 * a reference to the main object for a tethys export. I've no idea !
+	 * @param tethysExportParams2
+	 */
+	private void exportGeneralData(TethysExportParams tethysExportParams) {
+		// TODO Auto-generated method stub
+		
+	}
+
+	/**
+	 * Here is where we export data for a specific data stream to Tethys. 
+	 * 
+	 * @param aDataBlock
+	 * @param tethysExportParams
+	 * @param streamExportParams
+	 */
+	private void exportDataStream(PamDataBlock aDataBlock, TethysExportParams tethysExportParams,
+			StreamExportParams streamExportParams) {
+		/**
+		 * This will probably need to be passed additional parameters and may also want to return something
+		 * other than void in order to build a bigger Tethys document. 
+		 */
+		/*
+		 * Some examples of how to do whatever is needed to get schema and data out of PAMGuard. 
+		 */
+		/*
+		 * first we'll probably want a reference to the module containing the data.
+		 * in principle this can't get null, since the datablock was found be searching in 
+		 * the other direction. 
+		 */
+		PamControlledUnit pamControlledUnit = aDataBlock.getParentProcess().getPamControlledUnit();
+		
+		TethysDataProvider dataProvider = aDataBlock.getTethysDataProvider();
+
+		PamguardXMLWriter pamXMLWriter = PamguardXMLWriter.getXMLWriter();
+		
+		if (dataProvider == null) {
+			return;
+		}
+		
+		TethysSchema tethysSchema = dataProvider.getSchema();
+		/* 
+		 * the schema should have a Document object in it. If we wanted to turn 
+		 * that into an XML string we can ... 
+		 * (though I'd assume that having the Document is more useful)
+		 */
+		String schemaXMLString = pamXMLWriter.getAsString(tethysSchema.getXsd(), false);
+//		System.out.printf("Schema for %s is %s\n", aDataBlock.getDataName(), schemaXMLString);
+		
+		
+		/*
+		 * Get the XML settings for that datablock. This is (or should be
+		 * the parameters that were controlling that module, with adequate data about 
+		 * upstream modules). I think this has to go somewhere into the Detections document. 
+		 */
+		Document doc = pamXMLWriter.writeOneModule(pamXMLWriter, System.currentTimeMillis());
+		String moduleXML = null;
+		if (doc != null) {
+			// this string should be XML of all the settings for the module controlling this datablock. 
+			moduleXML = pamXMLWriter.getAsString(doc, true); // change to false to get smaller xml
+			System.out.printf("Module settings for datablock %s are:\n", moduleXML);
+			System.out.println(moduleXML);
+		}
+		
+		
+
+		
+		/**
+		 * Now can go through the data. Probably, we'll want to go through all the data in 
+		 * the project, but we can hold off on that for now and just go for data that 
+		 * are in memory. We'll also have to think a lot about updating parts of the 
+		 * database which have been reprocessed - what we want to do, should eventually all
+		 * be options set in the dialog and available within TethysExportParams
+		 * For now though, we're just going to export data that are in memory. 
+		 * Once basic export is working, I can easily enough write something which will go 
+		 * through an entire data set, go through between two times, etc. 
+		 */
+		// so this is a way of iterating through the data that are in memory, which will do for now ..
+		// do it with a data copy which can avoid synchronising the entire block for what may be a long time
+		// the copy function is itself synched, and is quite fast, so easier and safe this way
+		ArrayList<PamDataUnit> dataCopy = aDataBlock.getDataCopy();
+		for (PamDataUnit aData : dataCopy) {
+			// then we do whatever we need to do to convert this into 
+		}
+		
+		
+	}
+
+}

src/tethys/output/swing/TethysExportDialog.java

@@ -6,7 +6,9 @@ import java.awt.GridBagLayout;
 import java.awt.Window;
 import java.util.ArrayList;
 
+import javax.swing.JButton;
 import javax.swing.JCheckBox;
+import javax.swing.JLabel;
 import javax.swing.JPanel;
 import javax.swing.border.TitledBorder;
 
@@ -14,6 +16,7 @@ import PamController.PamController;
 import PamView.dialog.PamDialog;
 import PamView.dialog.PamGridBagContraints;
 import PamguardMVC.PamDataBlock;
+import PamguardMVC.dataSelector.DataSelector;
 import tethys.TethysControl;
 import tethys.output.StreamExportParams;
 import tethys.output.TethysExportParams;
@@ -81,22 +84,35 @@ public class TethysExportDialog extends PamDialog {
 		streamsPanel.setLayout(new GridBagLayout());
 		GridBagConstraints c = new PamGridBagContraints();
 		dataStreamSets = findDataStreams();
+		streamsPanel.add(new JLabel(" Data Stream ", JLabel.CENTER), c);
+		c.gridx++;
+		streamsPanel.add(new JLabel(" Data Select ", JLabel.CENTER), c);
 		for (DataStreamSet aSet : dataStreamSets) {
-			streamsPanel.add(aSet.checkBox, c);
+			c.gridx = 0;
 			c.gridy++;
+			streamsPanel.add(aSet.checkBox, c);
+			// try to add a data selector
+			DataSelector dataSelector = aSet.dataBlock.getDataSelector(tethysControl.getDataSelectName(), false);
+			if (dataSelector != null) {
+				c.gridx++;
+				JButton button = dataSelector.getDialogButton(this);
+				if (button != null) {
+					streamsPanel.add(button, c);
+				}
+			}
 		}
 		pack();
 	}
 	
 	/**
-	 * Get a set of data blocks that have SQLLogging. 
-	 * @return
+	 * Get a set of data blocks that can provide Tethys data. 
+	 * @return datablocks which can provide Tethys data
 	 */
 	private ArrayList<DataStreamSet> findDataStreams() {
 		ArrayList<DataStreamSet> sets = new ArrayList<>();
 		ArrayList<PamDataBlock> allDataBlocks = PamController.getInstance().getDataBlocks();
 		for (PamDataBlock aDataBlock : allDataBlocks) {
-			if (aDataBlock.getLogging() != null) {
+			if (aDataBlock.getTethysDataProvider() != null) {
 				sets.add(new DataStreamSet(aDataBlock));
 			}
 		}

src/tethys/pamdata/AutoTethysProvider.java

@@ -0,0 +1,42 @@
+package tethys.pamdata;
+
+import org.w3c.dom.Document;
+
+import PamguardMVC.PamDataBlock;
+import PamguardMVC.PamDataUnit;
+import generalDatabase.DBSchemaWriter;
+import generalDatabase.SQLLogging;
+
+/**
+ * Automatically provides Tethys data based on the SQL database interface 
+ * for a data block. 
+ * @author dg50
+ *
+ */
+public class AutoTethysProvider implements TethysDataProvider {
+
+	private PamDataBlock pamDataBlock;
+
+	public AutoTethysProvider(PamDataBlock pamDataBlock) {
+		this.pamDataBlock = pamDataBlock;
+	}
+
+	@Override
+	public TethysSchema getSchema() {
+		SQLLogging logging = pamDataBlock.getLogging();
+		if (logging == null) {
+			return null;
+		}
+		DBSchemaWriter schemaWriter = new DBSchemaWriter();
+		Document doc = schemaWriter.generateDatabaseSchema(pamDataBlock, logging, logging.getTableDefinition());
+		TethysSchema schema = new TethysSchema(doc);
+		return schema;
+	}
+
+	@Override
+	public TethysDataPoint getDataPoint(PamDataUnit pamDataUnit) {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+}

src/tethys/pamdata/TethysDataPoint.java

@@ -0,0 +1,18 @@
+package tethys.pamdata;
+
+import PamguardMVC.PamDataUnit;
+
+/**
+ * This will be a unit of Tethys Data, e.g. a Detection. 
+ * Can it also be used for things like GPS data ? 
+ * @author dg50
+ *
+ */
+public class TethysDataPoint {
+
+	public TethysDataPoint(PamDataUnit dataUnit) {
+		// TODO Auto-generated constructor stub
+	}
+
+
+}

src/tethys/pamdata/TethysDataProvider.java

@@ -0,0 +1,32 @@
+package tethys.pamdata;
+
+import PamguardMVC.PamDataUnit;
+
+/**
+ * Any PAMGuard data stream which can provide Detection data to PAMGuard will 
+ * be able to return one of these. It will provide a schema and a function which 
+ * can turn individual data units into data formatted for Tethys. The nature of how these 
+ * work TBD. 
+ * @author dg50
+ *
+ */
+public interface TethysDataProvider {
+
+	/**
+	 * This gets the Tethys schema for this type of data in whatever
+	 * form we decide it's best stored in, an XML string, or what ? 
+	 * @return
+	 */
+	public TethysSchema getSchema();
+	
+
+	/**
+	 * This will convert a data unit for this provider into whatever format we need the 
+	 * data to be in for Tethys. Some base function but also bespoke stuff depending on the
+	 * data type. Will probably need writing for every module individually?
+	 * @param pamDataUnit
+	 * @return
+	 */
+	public TethysDataPoint getDataPoint(PamDataUnit pamDataUnit);
+	
+}

src/tethys/pamdata/TethysSchema.java

@@ -0,0 +1,32 @@
+package tethys.pamdata;
+
+import org.w3c.dom.Document;
+
+/**
+ * object for a Tethys Schema. This may just be a very simple 
+ * wrapper around an XML string, or a JAXB object or something, 
+ * but may get more sophisticated. TBD in discussions with SDSU
+ */
+public class TethysSchema {
+
+	private Document schemaDoc;
+
+	public TethysSchema(Document doc) {
+		this.setXsd(doc);
+	}
+
+	/**
+	 * @return the xsd
+	 */
+	public Document getXsd() {
+		return schemaDoc;
+	}
+
+	/**
+	 * @param xsd the xsd to set
+	 */
+	public void setXsd(Document xsd) {
+		this.schemaDoc = xsd;
+	}
+
+}