nordic-dev.net/PAMGuard
nordic-dev.net/PAMGuard
2
0
Fork 0
mirror of https://github.com/PAMGuard/PAMGuard.git synced 2024-11-21 22:52:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding comments to table definitions.

Browse Source 
Update tabledefs so tableitems can have descriptions. Useful to the
coder, but can also export formal xms schema for each table.
This commit is contained in:
Douglas Gillespie 2022-11-07 17:15:04 +00:00
parent 3e717bd207
commit a72e68c26a
16 changed files with 386 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
src/Acquisition/AcquisitionLogging.java
View File

@ -33,23 +33,23 @@ public class AcquisitionLogging extends SQLLogging {
this.acquisitionControl = acquisitionControl;
tableDef = new PamTableDefinition(pamDataBlock.getDataName(), UPDATE_POLICY_OVERWRITE);
tableDef.addTableItem(adcClockTime = new PamTableItem("ADC Clock", Types.TIMESTAMP));
tableDef.addTableItem(adcClockMillis = new PamTableItem("ADC Clock millis", Types.INTEGER));
tableDef.addTableItem(rawADCTime = new PamTableItem("RAW ADC Clock", Types.TIMESTAMP));
tableDef.addTableItem(gSamples = new PamTableItem("GigaSamples", Types.INTEGER));
tableDef.addTableItem(samples = new PamTableItem("Samples", Types.INTEGER));
tableDef.addTableItem(GPSPPSTime = new PamTableItem("GPSPPSTime", Types.TIMESTAMP));
tableDef.addTableItem(status = new PamTableItem("Status", Types.CHAR, 20));
tableDef.addTableItem(reason = new PamTableItem("Reason", Types.CHAR, 50));
tableDef.addTableItem(daqSystemType = new PamTableItem("SystemType", Types.CHAR, 50));
tableDef.addTableItem(daqSystemName = new PamTableItem("SystemName", Types.CHAR, 50));
tableDef.addTableItem(sampleRate = new PamTableItem("sampleRate", Types.INTEGER));
tableDef.addTableItem(nChannels = new PamTableItem("nChannels", Types.INTEGER));
tableDef.addTableItem(voltsPeak2Peak = new PamTableItem("voltsPeak2Peak", Types.DOUBLE));
tableDef.addTableItem(gain = new PamTableItem("gain", Types.REAL));
tableDef.addTableItem(duration = new PamTableItem("duration", Types.DOUBLE));
tableDef.addTableItem(clockError = new PamTableItem("clockError", Types.DOUBLE));
tableDef.addTableItem(serverTime = new PamTableItem("Server Time", Types.TIMESTAMP));
tableDef.addTableItem(adcClockTime = new PamTableItem("ADC Clock", Types.TIMESTAMP, "Time based on PC clock and samples since last PC clock read"));
tableDef.addTableItem(adcClockMillis = new PamTableItem("ADC Clock millis", Types.INTEGER, "Millis for adcClockTime if not supported by database"));
tableDef.addTableItem(rawADCTime = new PamTableItem("RAW ADC Clock", Types.TIMESTAMP, "Time based on start time and number of samples acquired"));
tableDef.addTableItem(gSamples = new PamTableItem("GigaSamples", Types.INTEGER, "Billions of samples. Add this*1e9 to samples"));
tableDef.addTableItem(samples = new PamTableItem("Samples", Types.INTEGER, "Samples. Add this to GigaSamples*1e9"));
tableDef.addTableItem(GPSPPSTime = new PamTableItem("GPSPPSTime", Types.TIMESTAMP, "Time derived from a GPS with PPS output"));
tableDef.addTableItem(status = new PamTableItem("Status", Types.CHAR, 20, "Acquisition Status: 'Start', 'Stop', 'Continue'"));
tableDef.addTableItem(reason = new PamTableItem("Reason", Types.CHAR, 50, "Reason for status change"));
tableDef.addTableItem(daqSystemType = new PamTableItem("SystemType", Types.CHAR, 50, "Type of acquisition system"));
tableDef.addTableItem(daqSystemName = new PamTableItem("SystemName", Types.CHAR, 50, "Specific acquistision system"));
tableDef.addTableItem(sampleRate = new PamTableItem("sampleRate", Types.INTEGER, "Sample rate (samples per second)"));
tableDef.addTableItem(nChannels = new PamTableItem("nChannels", Types.INTEGER, "Total number of channels"));
tableDef.addTableItem(voltsPeak2Peak = new PamTableItem("voltsPeak2Peak", Types.DOUBLE, "Peak to peak input voltage of ADC"));
tableDef.addTableItem(gain = new PamTableItem("gain", Types.REAL, "Additional amplifier gain"));
tableDef.addTableItem(duration = new PamTableItem("duration", Types.DOUBLE, "Time since start in seconds"));
tableDef.addTableItem(clockError = new PamTableItem("clockError", Types.DOUBLE, "Clock error in milliseconds"));
tableDef.addTableItem(serverTime = new PamTableItem("Server Time", Types.TIMESTAMP, "Time taken from an Internet time server"));
setTableDefinition(tableDef);
}

8
src/Acquisition/AcquisitionProcess.java
View File

@ -254,9 +254,11 @@ public class AcquisitionProcess extends PamProcess {
bufferOverflow = false;
// daqCheckTime = PamCalendar.getTimeInMillis();
Timer t = new Timer(1, new ReallyStart());
t.setRepeats(false);
t.start();
if (!netRX) {
Timer t = new Timer(1, new ReallyStart());
t.setRepeats(false);
t.start();
}
}
private boolean addServerTime(DaqStatusDataUnit daqStatusDataUnit) {

3
src/Array/ThreadingHydrophoneLocator.java
View File

@ -77,6 +77,9 @@ public class ThreadingHydrophoneLocator extends StraightHydrophoneLocator implem
*/
while (gpsIt.hasPrevious()) {
gpsUnit = gpsIt.previous();
if (gpsUnit == null) {
continue;
}
gpsData = gpsUnit;
if (gpsUnit.getTimeInMillis() < timeMilliseconds) {
break;

4
src/PamController/command/SummaryCommand.java
View File

@ -1,6 +1,5 @@
package PamController.command;
import Map.GetMapFile;
import PamController.PamControlledUnit;
import PamController.PamController;
import PamUtils.PamCalendar;
@ -56,6 +55,9 @@ public class SummaryCommand extends ExtCommand {
totalString += String.format("\n<%s>%s:%s<\\%s>", aModule.getShortUnitType(),
aModule.getUnitName(), aString, aModule.getShortUnitType());
}
if (clear) {
lastCallTime = nowTime;
}
return totalString;
}

46
src/PamUtils/XMLUtils.java
View File

@ -1,7 +1,9 @@
package PamUtils;
import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
@ -9,6 +11,7 @@ import java.io.StringWriter;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
@ -183,4 +186,47 @@ public class XMLUtils {
}
return doc;
}
/**
* Write to the given file.
* @param doc xml document
* @param outFile file
* @throws IOException
*/
public static void writeToFile(Document doc, File outFile) throws IOException {
String asString = getAsString(doc);
if (asString!=null) {
try {
BufferedWriter out = new BufferedWriter(new FileWriter(outFile, false));
out.write(asString);
out.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* Get the xml document as a String.
* @param doc xml document
* @return xml content as a a string.
*/
public static String getAsString(Document doc) {
try {
DOMSource domSource = new DOMSource(doc);
StringWriter writer = new StringWriter();
StreamResult result = new StreamResult(writer);
TransformerFactory tf = TransformerFactory.newInstance();
Transformer transformer = tf.newTransformer();
transformer.setOutputProperty(OutputKeys.METHOD, "xml");
transformer.setOutputProperty(OutputKeys.ENCODING, "ISO-8859-1");
// transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
transformer.setOutputProperty(OutputKeys.INDENT, "yes");
transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "4");
transformer.transform(domSource, result);
return writer.toString();
} catch (TransformerException e) {
e.printStackTrace();
return null;
}
}
}

24
src/binaryFileStorage/BinaryStorageDialogPanel.java
View File

@ -130,7 +130,18 @@ public class BinaryStorageDialogPanel {
binaryStoreSettings.autoNewFiles = autoNewFiles.isSelected();
binaryStoreSettings.datedSubFolders = dateSubFolders.isSelected();
binaryStoreSettings.limitFileSize = limitfileSize.isSelected();
if (allowChannelOffsets) {
try {
binaryStoreSettings.channelShift = Integer.valueOf(channelOffset.getText());
}
catch (NumberFormatException e) {
return PamDialog.showWarning(owner, errorTitle, "Invalid channel offset number format");
}
if (binaryStoreSettings.channelShift < 0 || binaryStoreSettings.channelShift > 31) {
return PamDialog.showWarning(owner, errorTitle, "Channel offset bust be between 0 and 31");
}
}
if (PamController.getInstance().getRunMode() == PamController.RUN_PAMVIEW) {
return true;
}
@ -157,17 +168,6 @@ public class BinaryStorageDialogPanel {
return PamDialog.showWarning(owner, errorTitle, "Invalid file size data");
}
}
if (allowChannelOffsets) {
try {
binaryStoreSettings.channelShift = Integer.valueOf(channelOffset.getText());
}
catch (NumberFormatException e) {
return PamDialog.showWarning(owner, errorTitle, "Invalid channel offset number format");
}
if (binaryStoreSettings.channelShift < 0 || binaryStoreSettings.channelShift > 31) {
return PamDialog.showWarning(owner, errorTitle, "Channel offset bust be between 0 and 31");
}
}
NoiseStoreType nst = (NoiseStoreType) noiseStoreType.getSelectedItem();
if (nst == null) {
return PamDialog.showWarning(owner, errorTitle, "You must select a noise storage type");

15
src/generalDatabase/DBControl.java
View File

@ -428,11 +428,26 @@ PamSettingsSource {
JMenuItem speedMenu = new JMenuItem("Test database speed");
speedMenu.addActionListener(new SpeedMenu(parentFrame));
menu.add(speedMenu);
JMenuItem exSchema = new JMenuItem("Export database schema");
exSchema.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
exportDatabaseSchema(parentFrame);
}
});
menu.add(exSchema);
}
return menu;
}
protected void exportDatabaseSchema(JFrame parentFrame) {
dbProcess.exportDatabaseSchema(parentFrame);
}
class DatabaseFileMenuAction implements ActionListener {
private Frame frame;

42
src/generalDatabase/DBProcess.java
View File

@ -22,6 +22,8 @@ import java.sql.Statement;
import java.sql.Types;
import java.util.ArrayList;
import javax.swing.JFileChooser;
import javax.swing.JFrame;
import javax.swing.JOptionPane;
import javax.swing.SwingUtilities;
import javax.swing.Timer;
@ -36,7 +38,9 @@ import loggerForms.UDFTableDefinition;
import loggerForms.formdesign.FormEditor;
import PamController.PamControlledUnit;
import PamController.PamController;
import PamController.PamFolders;
import PamUtils.PamCalendar;
import PamUtils.PamFileChooser;
import PamView.dialog.warn.WarnOnce;
import PamguardMVC.PamDataBlock;
import PamguardMVC.PamDataUnit;
@ -1393,4 +1397,42 @@ public class DBProcess extends PamProcess {
return logViewerSettings;
}
/**
* Export all available database schema, converting PamTableDefinitions into
* valid xsd documents.
* @param parentFrame
*/
public void exportDatabaseSchema(JFrame parentFrame) {
File startLoc = PamFolders.getFileChooserPath(PamFolders.getDefaultProjectFolder());
PamFileChooser fc = new PamFileChooser();
fc.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY);
startLoc = PamFolders.getFileChooserPath(startLoc);
fc.setCurrentDirectory(startLoc);
int ans = fc.showDialog(parentFrame, "Select storage folder");
if (ans == JFileChooser.APPROVE_OPTION) {
startLoc = fc.getSelectedFile();
exportDatabaseSchema(parentFrame, startLoc);
}
}
/**
* Export all table definitions to xsd files in given folder.
* @param parentFrame
* @param folder
*/
private void exportDatabaseSchema(JFrame parentFrame, File folder) {
ArrayList<PamDataBlock> allDataBlocks = PamController.getInstance().getDataBlocks();
DBSchemaWriter schemaWriter = new DBSchemaWriter();
for (PamDataBlock aBlock : allDataBlocks) {
SQLLogging logging = aBlock.getLogging();
if (logging == null) {
continue;
}
schemaWriter.writeSchema(folder, aBlock);
}
}
}

138
src/generalDatabase/DBSchemaWriter.java Normal file
View File

@ -0,0 +1,138 @@
package generalDatabase;
import java.io.File;
import java.io.IOException;
import java.sql.Types;
import javax.xml.parsers.DocumentBuilderFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import PamUtils.XMLUtils;
import PamguardMVC.PamDataBlock;
/**
* Functions for writing database table schema in a format
* compatible with the Tethys schemas.
* @author dg50
*
*/
public class DBSchemaWriter {
public DBSchemaWriter() {
// TODO Auto-generated constructor stub
}
public boolean writeSchema(File outputFolder, PamDataBlock dataBlock) {
if (dataBlock == null) {
return false;
}
SQLLogging logging = dataBlock.getLogging();
if (logging == null) {
return false;
}
PamTableDefinition tableDef = logging.getTableDefinition();
exportDatabaseSchema(outputFolder, dataBlock, tableDef);
return true;
}
private void exportDatabaseSchema(File outputFolder, PamDataBlock dataBlock, PamTableDefinition tableDef) {
String tableName = tableDef.getTableName();
File outputFile = new File(outputFolder, tableName+".xsd");
Document doc = PamUtils.XMLUtils.createBlankDoc();
Element el = doc.createElement("xs:schema");
el.setAttribute("xmlns:xs","http://www.w3.org/2001/XMLSchema");
el.setAttribute("targetNamespace", "http://tethys.sdsu.edu/schema/1.0");
doc.appendChild(el);
for (PamTableItem tableItem : tableDef.pamTableItems) {
Element itemEl = doc.createElement("xs:element");
itemEl.setAttribute("name", tableItem.getName());
itemEl.setAttribute("type", sqlTypeToString(tableItem.getSqlType(), tableItem.getLength()));
String documentation = tableItem.getDescription();
if (documentation != null) {
Element annotation = doc.createElement("xs:annotation");
itemEl.appendChild(annotation);
Element docEl = doc.createElement("xs:documentation");
docEl.setTextContent(documentation);
annotation.appendChild(docEl);
}
el.appendChild(itemEl);
}
try {
XMLUtils.writeToFile(doc, outputFile);
} catch (IOException e) {
e.printStackTrace();
}
}
private String sqlTypeToString(int sqlType, int length) {
switch (sqlType) {
case Types.ARRAY:
return "ARRAY";
case Types.BIGINT:
return "xs:long";
case Types.BINARY:
return "BINARY";
case Types.BIT:
return "BIT";
case Types.BLOB:
return "BLOB";
case Types.BOOLEAN:
return "xs:boolean";
case Types.CHAR:
return "xs:string";
case Types.CLOB:
return "CLOB";
case Types.DATALINK:
return "DATALINK";
case Types.DATE:
return "xs:dateTime";
case Types.DECIMAL:
return "DECIMAL";
case Types.DISTINCT:
return "DISTINCT";
case Types.DOUBLE:
return "xs:double";
case Types.FLOAT:
return "xs:float";
case Types.INTEGER:
return "xs:int";
case Types.JAVA_OBJECT:
return "JAVA_OBJECT";
case Types.LONGVARBINARY:
return "LONGVARBINARY(" + length + ")";
case Types.LONGVARCHAR:
return "LONGVARCHAR(" + length + ")";
case Types.NULL:
return "NULL";
case Types.NUMERIC:
return "NUMERIC";
case Types.OTHER:
return "OTHER";
case Types.REAL:
return "xs:float";
case Types.REF:
return "REF";
case Types.SMALLINT:
return "xs:short";
case Types.STRUCT:
return "STRUCT";
case Types.TIME:
return "TIME";
case Types.TIMESTAMP:
return "xs:dateTime";
case Types.TINYINT:
return "TINYINT";
case Types.VARBINARY:
return "VARBINARY(" + length + ")";
case Types.VARCHAR:
return "VARCHAR(" + length + ")";
}
return null;
}
}

16
src/generalDatabase/PamTableDefinition.java
View File

@ -55,15 +55,15 @@ public class PamTableDefinition extends EmptyTableDefinition implements Cloneabl
*/
public PamTableDefinition(String tableName, int updatePolicy) {
super(tableName, updatePolicy);
pamTableItems.add(uid = new PamTableItem("UID", Types.BIGINT));
pamTableItems.add(timeStampItem = new PamTableItem(utcColName, Types.TIMESTAMP));
pamTableItems.add(timeStampMillis = new PamTableItem("UTCMilliseconds", Types.SMALLINT));
pamTableItems.add(localTime = new PamTableItem("PCLocalTime", Types.TIMESTAMP));
pamTableItems.add(pcTime = new PamTableItem("PCTime", Types.TIMESTAMP));
pamTableItems.add(channelBitmap = new PamTableItem("ChannelBitmap", Types.INTEGER));
pamTableItems.add(sequenceBitmap = new PamTableItem("SequenceBitmap", Types.INTEGER));
pamTableItems.add(uid = new PamTableItem("UID", Types.BIGINT, "Unique Identifier"));
pamTableItems.add(timeStampItem = new PamTableItem(utcColName, Types.TIMESTAMP, "Timestamp UTC"));
pamTableItems.add(timeStampMillis = new PamTableItem("UTCMilliseconds", Types.SMALLINT, "Time milliseconds (for databases which do not support millis)"));
pamTableItems.add(localTime = new PamTableItem("PCLocalTime", Types.TIMESTAMP, "Local time on PC"));
pamTableItems.add(pcTime = new PamTableItem("PCTime", Types.TIMESTAMP, "Time data written, UTC. Same as UTC for real time data, current time for offline file analysis"));
pamTableItems.add(channelBitmap = new PamTableItem("ChannelBitmap", Types.INTEGER, "Bitmap of input channels used"));
pamTableItems.add(sequenceBitmap = new PamTableItem("SequenceBitmap", Types.INTEGER, "Bitmap of beam or channel outputs used"));
if (updatePolicy == SQLLogging.UPDATE_POLICY_WRITENEW) {
addTableItem(updateReference = new PamTableItem("UpdateOf", Types.INTEGER));
addTableItem(updateReference = new PamTableItem("UpdateOf", Types.INTEGER, "Reference to previous value"));
}
}

66
src/generalDatabase/PamTableItem.java
View File

@ -50,6 +50,14 @@ public class PamTableItem implements Cloneable {
*/
private Object value;
/**
* Test description of field. Can be exported to schema and may
* be possible to write into database description fields one day. In any case
* it can be useful to the programmer when looking back to see what on earth
* code is supposed to be doing.
*/
private String description;
/*
* Reference to another PamTableItem in a different
* table. This must be of the same sqlType as this
@ -60,6 +68,11 @@ public class PamTableItem implements Cloneable {
*/
private PamTableItem crossReferenceItem;
/**
* Generate a table item
* @param name name of table item
* @param sqlType SQL Type from java.sql.Types, e.g. Types.INTEGER
*/
public PamTableItem(String name, int sqlType) {
super();
this.name = name;
@ -67,7 +80,28 @@ public class PamTableItem implements Cloneable {
this.length = 0;
this.required = false;
}
/**
* Generate a table item
* @param name name of table item
* @param sqlType SQL Type from java.sql.Types, e.g. Types.INTEGER
* @param description optional description
*/
public PamTableItem(String name, int sqlType, String description) {
super();
this.name = name;
this.sqlType = sqlType;
this.description = description;
this.length = 0;
this.required = false;
}
/**
* Generate a table item
* @param name name of table item
* @param sqlType SQL Type from java.sql.Types, e.g. Types.INTEGER
* @param length length (only applicable to Types.CHAR types)
*/
public PamTableItem(String name, int sqlType, int length) {
super();
this.name = name;
@ -76,6 +110,22 @@ public class PamTableItem implements Cloneable {
this.required = false;
}
/**
* Generate a table item
* @param name name of table item
* @param sqlType SQL Type from java.sql.Types, e.g. Types.INTEGER
* @param length length (only applicable to Types.CHAR types)
* @param description optional description
*/
public PamTableItem(String name, int sqlType, int length, String description) {
super();
this.name = name;
this.sqlType = sqlType;
this.length = length;
this.description = description;
this.required = false;
}
public PamTableItem(String name, int sqlType, int length, boolean required) {
super();
this.name = name;
@ -500,6 +550,20 @@ public class PamTableItem implements Cloneable {
return null;
}
}
/**
* @return the description
*/
public String getDescription() {
return description;
}
/**
* @param description the description to set
*/
public void setDescription(String description) {
this.description = description;
}
}

5
src/networkTransfer/receive/BuoyStatusData.java
View File

@ -144,4 +144,9 @@ public class BuoyStatusData implements Serializable, ManagedParameters {
return ps;
}
@Override
public String toString() {
return String.format("Buoy %d(%d) ip %s", buoyId1, buoyId2, ipAddr);
}
}

5
src/networkTransfer/receive/BuoyStatusDataUnit.java
View File

@ -387,6 +387,11 @@ public class BuoyStatusDataUnit extends PamDataUnit {
return buoyStatusData;
}
@Override
public String toString() {
return String.format("Status %s\n", buoyStatusData.toString());
}
// /**
// * @return the genericStringPairsChanged
// */

2
src/networkTransfer/receive/NetworkReceiver.java
View File

@ -770,7 +770,7 @@ public class NetworkReceiver extends PamControlledUnit implements PamSettings, N
}
private NetworkObject interpretPamCommand(NetworkObject receivedData, BuoyStatusDataUnit buoyStatusDataUnit) {
// System.out.println("Network command received: " + getPamCommandString(dataId2));
// System.out.println("Network command received: " + getPamCommandString(dataId2));
buoyStatusDataUnit.setCommandStatus(receivedData.getDataType2());
long time = PamCalendar.getTimeInMillis();
if (receivedData.getData() != null && receivedData.getDataLength() >= 8) {

10
src/whistlesAndMoans/WhistleMoanControl.java
View File

@ -183,11 +183,6 @@ public class WhistleMoanControl extends PamControlledUnit implements PamSettings
return true;
}
@Override
public String getModuleSummary() {
return whistleToneProcess.getModuleSummary();
}
@Override
public Object getShortUnitType() {
return "WMD";
@ -225,4 +220,9 @@ public class WhistleMoanControl extends PamControlledUnit implements PamSettings
}
return null;
}
@Override
public String getModuleSummary(boolean clear) {
return whistleToneProcess.getModuleSummary(clear);
}
}

27
src/whistlesAndMoans/WhistleToneConnectProcess.java
View File

@ -968,17 +968,6 @@ public class WhistleToneConnectProcess extends PamProcess {
}
public String getModuleSummary() {
String sumText = String.format("%d", NSUMMARYPOINTS);
for (int i = 0; i < NSUMMARYPOINTS; i++) {
sumText += String.format(",%d",whistleSummaryCount[i]);
}
clearSummaryData();
return sumText;
}
/**
* When delay data are written to binary files, int16's are used, but these
* must be scaled up to allow for sub-sample timing. How much they can be
@ -1099,4 +1088,20 @@ public class WhistleToneConnectProcess extends PamProcess {
return fftUnitsOut;
}
/*
* replicate Decimus function to give counts of whistles in four evenly spaced frequency
* bins.
*/
public String getModuleSummary(boolean clear) {
String sumText = String.format("%d", NSUMMARYPOINTS);
for (int i = 0; i < NSUMMARYPOINTS; i++) {
sumText += String.format(",%d",whistleSummaryCount[i]);
}
if (clear) {
clearSummaryData();
}
return sumText;
}
}