From 782057c020a26dbb8845124bcb43283d2f6b5a7d Mon Sep 17 00:00:00 2001 From: Douglas Gillespie <50671166+douggillespie@users.noreply.github.com> Date: Sat, 30 Dec 2023 17:28:23 +0000 Subject: [PATCH 1/2] claspath update --- .classpath | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.classpath b/.classpath index 188c7291..26c568d7 100644 --- a/.classpath +++ b/.classpath @@ -6,8 +6,9 @@ - + + From 52cc8c71775ce6e13d1ea500ccd60e0b04fed860 Mon Sep 17 00:00:00 2001 From: Douglas Gillespie <50671166+douggillespie@users.noreply.github.com> Date: Wed, 17 Jan 2024 21:24:04 +0000 Subject: [PATCH 2/2] Batch processing Adding functionality for offline task batch processing. --- src/PamController/PamControlledUnit.java | 14 +- src/PamController/PamController.java | 129 ++++++++++++++- src/PamController/PamSettingManager.java | 6 +- .../command/BatchStatusCommand.java | 14 ++ src/PamController/command/StatusCommand.java | 28 +--- .../settings/BatchViewSettingsImport.java | 155 ++++++++++++++++++ .../settings/SettingsImport.java | 5 +- src/PamModel/PamModel.java | 17 +- src/Spectrogram/SpectrogramDisplay.java | 12 +- src/binaryFileStorage/BinaryStore.java | 42 ++++- src/dataGram/DatagramManager.java | 17 +- src/generalDatabase/DBControlUnit.java | 15 +- src/noiseMonitor/NoiseControl.java | 12 ++ src/noiseMonitor/NoiseTabPanel.java | 9 +- src/offlineProcessing/OfflineTask.java | 10 ++ src/offlineProcessing/OfflineTaskManager.java | 55 ++++++- src/pamguard/GlobalArguments.java | 13 ++ src/pamguard/Pamguard.java | 10 ++ 18 files changed, 506 insertions(+), 57 deletions(-) create mode 100644 src/PamController/settings/BatchViewSettingsImport.java diff --git a/src/PamController/PamControlledUnit.java b/src/PamController/PamControlledUnit.java index f1962ca8..a8a8c96e 100644 --- a/src/PamController/PamControlledUnit.java +++ b/src/PamController/PamControlledUnit.java @@ -30,12 +30,8 @@ import javax.swing.JMenuBar; import javax.swing.JMenuItem; import javax.swing.JOptionPane; -import offlineProcessing.OfflineTask; import offlineProcessing.OfflineTaskGroup; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - import PamController.status.ModuleStatus; import PamController.status.ModuleStatusManager; import PamController.status.ProcessCheck; @@ -845,6 +841,16 @@ public abstract class PamControlledUnit implements SettingsNameProvider { } return worstStatus; } + + /** + * Get the offline state of this module. This can generally + * be idle, but can be a higher state when map making at startup + * and when running an offline task. + * @return + */ + public int getOfflineState() { + return PamController.PAM_IDLE; + } // /** // * Get a list of available offline tasks for this module. This is mostly used for tasks that diff --git a/src/PamController/PamController.java b/src/PamController/PamController.java index d74cd730..6398f9f1 100644 --- a/src/PamController/PamController.java +++ b/src/PamController/PamController.java @@ -37,6 +37,7 @@ import javax.swing.ToolTipManager; import com.jcraft.jsch.ConfigRepository.Config; import com.sun.xml.bind.v2.TODO; +import Acquisition.AcquisitionControl; import Acquisition.AcquisitionProcess; //import com.sun.org.apache.xerces.internal.dom.DocumentImpl; @@ -64,6 +65,7 @@ import PamController.command.TerminalController; import PamController.command.WatchdogComms; import PamController.fileprocessing.ReprocessManager; import PamController.masterReference.MasterReferencePoint; +import PamController.settings.BatchViewSettingsImport; import PamController.settings.output.xml.PamguardXMLWriter; import PamController.settings.output.xml.XMLWriterDialog; import PamController.soundMedium.GlobalMediumManager; @@ -123,6 +125,8 @@ public class PamController implements PamControllerInterface, PamSettings { public static final int PAM_INITIALISING = 4; public static final int PAM_STOPPING = 5; public static final int PAM_COMPLETE = 6; + public static final int PAM_MAPMAKING = 7; + public static final int PAM_OFFLINETASK = 8; // status' for RunMode = RUN_PAMVIEW public static final int PAM_LOADINGDATA = 2; @@ -234,6 +238,7 @@ public class PamController implements PamControllerInterface, PamSettings { */ private Thread statusCheckThread; private WaitDetectorThread detectorEndThread; + private boolean firstDataLoadComplete; private PamController(int runMode, Object object) { @@ -436,7 +441,7 @@ public class PamController implements PamControllerInterface, PamSettings { // addModule(mi, "Temporary Database"); // } - // Add a note to the putput console for the user to ignore the SLF4J warning (see http://www.slf4j.org/codes.html#StaticLoggerBinder + // Add a note to the output console for the user to ignore the SLF4J warning (see http://www.slf4j.org/codes.html#StaticLoggerBinder // for details). I spent a few hours trying to get rid of this warning, but without any luck. If you do a google search // there are a lot of forum suggestions on how to fix, but none seemed to work for me. Added both slf4j-nop and // slf4j-simple to dependency list, neither made a difference. Changed order of dependencies, ran purges and updates, @@ -490,17 +495,31 @@ public class PamController implements PamControllerInterface, PamSettings { addView(guiFrameManager.initPrimaryView(this, pamModelInterface)); } + /** + * Calling this will cause a callback to this.restoreSettings which + * includes a list of modules which will then get created, and in turn + * load all of their own settings from the settings manager. + */ PamSettingManager.getInstance().registerSettings(this); + + /** + * For offline batch processing a few funnies happen here. We'll be open + * in viewer mode, but it's likely a psf will have been passed as an input argument. + * We will therefore have to extract all the modules from that psfx as well and either + * add them as new modules, or get their settings and use those to update existing settings + * That should probably be done here before the final calls to setup processes, etc. + */ + if (getRunMode() == RUN_PAMVIEW && PamSettingManager.remote_psf != null) { + loadOtherSettings(PamSettingManager.remote_psf); + } + /* + * Get any other required modules for this run mode. + */ pamModelInterface.startModel(); setupProcesses(); - // if (getRunMode() == RUN_PAMVIEW) { - // createViewerStatusBar(); - // pamControlledUnits.add(new OfflineProcessingControlledUnit("OfflineProcessing")); - // } - /* * We are running as a remote application, start process straight away! */ @@ -571,6 +590,7 @@ public class PamController implements PamControllerInterface, PamSettings { // }); } + /** * Clear all data selectors and symbol managers. Required since some of these will have loaded as various modules were created, * but may also require additional data selectors and symbol managers from super detections which were not availble. @@ -583,6 +603,11 @@ public class PamController implements PamControllerInterface, PamSettings { } + /** + * This gets called after other data initialisation tasks (such as data mapping). + * @author dg50 + * + */ class DataInitialised implements Runnable { @Override public void run() { @@ -1765,6 +1790,10 @@ public class PamController implements PamControllerInterface, PamSettings { if (moduleChange(changeType)) { clearSelectorsAndSymbols(); } + + if (changeType == DATA_LOAD_COMPLETE) { + firstDataLoadComplete = true; + } } @@ -1901,12 +1930,75 @@ public class PamController implements PamControllerInterface, PamSettings { public void setPamStatus(int pamStatus) { this.pamStatus = pamStatus; + /* + * This only get's called once when set idle at viewer mode startup. + */ +// System.out.printf("******* PamController.setPamStatus to %d, real status is %d\n", pamStatus, getRealStatus()); if (getRunMode() != RUN_PAMVIEW) { TopToolBar.enableStartButton(pamStatus == PAM_IDLE); TopToolBar.enableStopButton(pamStatus == PAM_RUNNING); } showStatusWarning(pamStatus); } + + /** + * This was within the StatusCommand class, but useful to have it here since it's needed + * in more than one place. In viewer mode at startup there are a number of things going on + * in different threads, such as the creation of datamaps, and this can (hopefully) handle those bespoke + * goings on. + * @return program status for multithreaded statup tasks. + */ + public int getRealStatus() { + PamController pamController = PamController.getInstance(); + if (pamController.isInitializationComplete() == false) { + return PamController.PAM_INITIALISING; + } + int runMode = PamController.getInstance().getRunMode(); + if (runMode == PamController.RUN_NETWORKRECEIVER) { + return PamController.PAM_RUNNING; + } + int status = pamController.getPamStatus(); + if (status == PamController.PAM_IDLE) { + status = PamController.PAM_IDLE; + } + else { + ArrayList daqs = PamController.getInstance().findControlledUnits(AcquisitionControl.unitType); + if (daqs != null) for (int i = 0; i < daqs.size(); i++) { + try { + AcquisitionControl daq = (AcquisitionControl) daqs.get(i); + if (daq.isStalled()) { + status = PamController.PAM_STALLED; + } + } + catch (Exception e) { + e.printStackTrace(); + } + } + } + WatchdogComms watchdogComms = PamController.getInstance().getWatchdogComms(); + status = watchdogComms.getModifiedWatchdogState(status); + /* + * This function is now being used in batch processing of offline data, where it may be necessary + * to get status information from many different modules, for example when executing offline tasks + * or just at startup while generating datamaps and datagrams. + * So go through all modules and get the highest state of any of them. + */ + if (getRunMode() == RUN_PAMVIEW) { + if (firstDataLoadComplete == false) { + status = PAM_INITIALISING; + } + try { + for (PamControlledUnit aUnit : pamConfiguration.getPamControlledUnits()) { + status = Math.max(status, aUnit.getOfflineState()); + } + } + catch (Exception e) { + //just incase there is a concurrent modification at startup. + } + } + + return status; + } /** * show a warning when we're waiting for detectors to stop @@ -2191,6 +2283,31 @@ public class PamController implements PamControllerInterface, PamSettings { private boolean manualStop; + /** + * Used when in viewer mode and planning batch processing with a modified + * configuration, i.e. the command line has been supplied a normal viewer mode + * database and also a psfx file. The settings from the database will already have + * been loaded, this will load any modules that weren't there and will override all the + * settings in other modules with these ones (except some specials such as data storage locations) + * @param psfxFile Name of additional psfx file. + */ + private boolean loadOtherSettings(String psfxName) { + + File psfxFile = new File(psfxName); + if (psfxFile.exists() == false) { + return false; + } + + PamSettingsGroup settingsGroup = PSFXReadWriter.getInstance().loadFileSettings(psfxFile); + if (settingsGroup == null) { + return false; + } + + BatchViewSettingsImport importer = new BatchViewSettingsImport(this, settingsGroup); + importer.importSettings(); + return true; + } + /** * Called to load a specific set of PAMGUARD settings in * viewer mode, which were previously loaded in from a diff --git a/src/PamController/PamSettingManager.java b/src/PamController/PamSettingManager.java index 46bc0679..2e3178ce 100644 --- a/src/PamController/PamSettingManager.java +++ b/src/PamController/PamSettingManager.java @@ -44,6 +44,7 @@ import javax.swing.plaf.FontUIResource; import pamViewFX.fxNodes.utilsFX.PamUtilsFX; import pamViewFX.fxSettingsPanes.SettingsFileDialogFX; +import pamguard.GlobalArguments; //XMLSettings //import org.jdom.Document; @@ -492,7 +493,7 @@ public class PamSettingManager { /** * Call just before PAMGUARD exits to save the settings * either to psf and / or database tables. - * @return true if settings saved sucessfully. + * @return true if settings saved successfully. */ public boolean saveFinalSettings() { int runMode = PamController.getInstance().getRunMode(); @@ -1029,8 +1030,9 @@ public class PamSettingManager { loadingLocalSettings = true; loadSettingsFileData(); + - if (PamSettingManager.RUN_REMOTE == false) { + if (PamSettingManager.RUN_REMOTE == false && GlobalArguments.isBatch() == false) { if (settingsFileData != null) { TipOfTheDayManager.getInstance().setShowAtStart(settingsFileData.showTipAtStartup); if (settingsFileData.showTipAtStartup) { diff --git a/src/PamController/command/BatchStatusCommand.java b/src/PamController/command/BatchStatusCommand.java index a58012b2..046dff3e 100644 --- a/src/PamController/command/BatchStatusCommand.java +++ b/src/PamController/command/BatchStatusCommand.java @@ -2,6 +2,7 @@ package PamController.command; import Acquisition.AcquisitionControl; import PamController.PamController; +import offlineProcessing.OfflineTaskManager; import pamViewFX.PamControlledGUIFX; /** @@ -21,6 +22,19 @@ public class BatchStatusCommand extends ExtCommand { @Override public String execute(String command) { + if (PamController.getInstance().getRunMode() == PamController.RUN_NORMAL) { + return getNormalModeStatus(command); + } + else { + return getViewerModeStatus(command); + } + } + + private String getViewerModeStatus(String command) { + return OfflineTaskManager.getManager().getBatchStatus(); + } + + private String getNormalModeStatus(String command) { AcquisitionControl daqControl = (AcquisitionControl) PamController.getInstance().findControlledUnit(AcquisitionControl.class, null); if (daqControl == null) { return null; diff --git a/src/PamController/command/StatusCommand.java b/src/PamController/command/StatusCommand.java index 7d500e95..74acf77f 100644 --- a/src/PamController/command/StatusCommand.java +++ b/src/PamController/command/StatusCommand.java @@ -31,33 +31,7 @@ public class StatusCommand extends ExtCommand { private int getRealStatus() { PamController pamController = PamController.getInstance(); - if (pamController.isInitializationComplete() == false) { - return PamController.PAM_INITIALISING; - } - int runMode = PamController.getInstance().getRunMode(); - if (runMode == PamController.RUN_NETWORKRECEIVER) { - return PamController.PAM_RUNNING; - } - int status = pamController.getPamStatus(); - if (status == PamController.PAM_IDLE) { - status = PamController.PAM_IDLE; - } - else { - ArrayList daqs = PamController.getInstance().findControlledUnits(AcquisitionControl.unitType); - if (daqs != null) for (int i = 0; i < daqs.size(); i++) { - try { - AcquisitionControl daq = (AcquisitionControl) daqs.get(i); - if (daq.isStalled()) { - status = PamController.PAM_STALLED; - } - } - catch (Exception e) { - e.printStackTrace(); - } - } - } - WatchdogComms watchdogComms = PamController.getInstance().getWatchdogComms(); - return watchdogComms.getModifiedWatchdogState(status); + return pamController.getRealStatus(); } diff --git a/src/PamController/settings/BatchViewSettingsImport.java b/src/PamController/settings/BatchViewSettingsImport.java new file mode 100644 index 00000000..d9cb5b2a --- /dev/null +++ b/src/PamController/settings/BatchViewSettingsImport.java @@ -0,0 +1,155 @@ +package PamController.settings; + +import java.util.ArrayList; + +import PamController.DataInputStore; +import PamController.OfflineDataStore; +import PamController.OfflineFileDataStore; +import PamController.PamControlledUnit; +import PamController.PamControlledUnitSettings; +import PamController.PamController; +import PamController.PamSettingManager; +import PamController.PamSettings; +import PamController.PamSettingsGroup; +import PamController.UsedModuleInfo; +import PamModel.PamModuleInfo; + +/** + * A set of functions to handle importing and overriding settings imported from a psfx during + * batch mode processing in viewer mode. Some of this is quite similar to code in SettingsImport + * but different enough that it's easier to have in a separate set of functions. + * @author dg50 + * + */ +public class BatchViewSettingsImport { + + private PamController pamController; + private PamSettingsGroup settingsGroup; + private SettingsImport settingsImport; + + public BatchViewSettingsImport(PamController pamController, PamSettingsGroup settingsGroup) { + this.pamController = pamController; + this.settingsGroup = settingsGroup; + settingsImport = new SettingsImport(pamController); + } + + public boolean importSettings() { + + // first organise by controlled unit + ArrayList moduleGroups = settingsImport.organiseSettingsGroups(settingsGroup.getUnitSettings()); + // can now go through those modules and see which exist and which need to be added. Not asking questions, just doing it! + for (SettingsImportGroup moduleGroup : moduleGroups) { + UsedModuleInfo moduleInfo = moduleGroup.getUsedModuleInfo(); + PamControlledUnit exModule = pamController.findControlledUnit(moduleInfo.getUnitType(), moduleInfo.getUnitName()); + boolean existingStore = false; + if (exModule != null) { + existingStore = isDataStore(exModule); + System.out.printf("Module %s:%s already exists in model (data store: %s)\n", moduleInfo.getUnitType(), moduleInfo.getUnitName(), Boolean.valueOf(existingStore)); + } + else { + System.out.printf("Module %s:%s will be added to model\n", moduleInfo.getUnitType(), moduleInfo.getUnitName()); + // add the module. No questions asked. + PamModuleInfo pamModuleInfo = moduleGroup.getPamModuleInfo(); + if (pamModuleInfo == null) { + System.out.printf("Module %s:%s is not available to this PAMGuard installation\n", moduleInfo.getUnitType(), moduleInfo.getUnitName()); + continue; + } + exModule = pamController.addModule(pamModuleInfo, moduleInfo.getUnitName()); + } + // set the settings for that module, but only if it's NOT a data storage module. + if (exModule == null) { + continue; + } +// if (exModule.getUnitName().contains("Noise")) { +// System.out.printf("restoring settings for %s, %s\n", exModule.getUnitType(), exModule.getUnitName()); +// +// } + if (isDataStore(exModule)) { + System.out.printf("Skip restoring settings for %s, %s\n", exModule.getUnitType(), exModule.getUnitName()); + continue; + } + restoreSettings(moduleGroup.getMainSettings()); + ArrayList subSets = moduleGroup.getSubSettings(); + if (subSets != null) { + for (PamControlledUnitSettings aSet : subSets) { + restoreSettings(aSet); + } + } +// exModule.notifyModelChanged(PamController.INITIALIZATION_COMPLETE); +// if (exModule.getTabPanel() != null) { +//// exModule.getTabPanel() +// } + } + /* + * Don't need to call this here since it get's called shortly + * from PAMController once all modules are in place. + */ +// pamController.notifyModelChanged(PamController.INITIALIZATION_COMPLETE); + + // send out an initialisation complete to help restore settings +// ArrayList allSettings = settingsGroup.getUnitSettings(); +// for (PamControlledUnitSettings aSet : allSettings) { +// PamSettings owner = PamSettingManager.getInstance().findSettingsOwner(aSet.getUnitType(), aSet.getUnitName(), null); +// if (owner == null) { +// System.out.printf("Cannot find owner for %s, %s, %s\n", aSet.getUnitType(), aSet.getUnitName(), aSet.getOwnerClassName()); +// } +// else { +// owner.restoreSettings(aSet); +// } +// } + + return true; + } + + /** + * Find the owner of these settings and send it it's new settings. + * @param aSet + * @return true if found and set sucessfully. + */ + private boolean restoreSettings(PamControlledUnitSettings aSet) { + PamSettings owner = PamSettingManager.getInstance().findSettingsOwner(aSet.getUnitType(), aSet.getUnitName(), null); + if (owner == null) { + System.out.printf("Cannot find owner for %s, %s, %s\n", aSet.getUnitType(), aSet.getUnitName(), aSet.getOwnerClassName()); + return false; + } + else { + try { + owner.restoreSettings(aSet); + } + catch (Exception e) { + System.out.printf("Exception restoring settings %s, %s, %s\n", aSet.getUnitType(), aSet.getUnitName(), aSet.getOwnerClassName()); + e.printStackTrace(); + return false; + } + } + return true; + } + + + + /** + * Is the module a data store. If it is, we probably won't want to copy over it's settings. + * @param controlledUnit PAMGuard module + * @return true if it's data output or input. + */ + private boolean isDataStore(PamControlledUnit controlledUnit) { + return isDataStore(controlledUnit.getClass()); + } + + /** + * Is the class a data store. If it is, we probably won't want to copy over it's settings. + * @param moduleClass module class + * @return true if it's data output or input. + */ + private boolean isDataStore(Class moduleClass) { + //OfflineFileDataStore, DataInputStore + if (OfflineDataStore.class.isAssignableFrom(moduleClass)) { + return true; + } + if (DataInputStore.class.isAssignableFrom(moduleClass)) { + return true; + } + return false; + } + +} diff --git a/src/PamController/settings/SettingsImport.java b/src/PamController/settings/SettingsImport.java index 16e23f35..e5ed64d6 100644 --- a/src/PamController/settings/SettingsImport.java +++ b/src/PamController/settings/SettingsImport.java @@ -29,7 +29,10 @@ import PamView.dialog.warn.WarnOnce; */ public class SettingsImport { + private PamController pamController; + public SettingsImport(PamController pamController) { + this.pamController = pamController; } /** @@ -235,7 +238,7 @@ public class SettingsImport { * @param settings * @return */ - ArrayList organiseSettingsGroups(ArrayList settings) { + public ArrayList organiseSettingsGroups(ArrayList settings) { /** * this needs rewriting for psfx files which are organised differently. first we need to find * a list of PAMGuard modules by finding the settings group of the PAMController. diff --git a/src/PamModel/PamModel.java b/src/PamModel/PamModel.java index 9cdb9992..3b0aa6d0 100644 --- a/src/PamModel/PamModel.java +++ b/src/PamModel/PamModel.java @@ -926,14 +926,17 @@ final public class PamModel implements PamSettings { } - /* (non-Javadoc) - * @see PamModel.PamModelInterface#startModel() + + + /** + * Add any remaining REQUIRED modules.
+ * this get's called after the PamController has loaded it's main settings. + * So at this point, go through all the PamModuleInfo's and check that + * all have at least the minimum number required + * @return true */ public synchronized boolean startModel() { /* - * this get's called after the PamController has loaded it's main settings. - * So at this point, go through all the PamModuleInfo's and check that - * all have at least the minimum number required */ PamSettingManager.getInstance().registerSettings(this); @@ -950,11 +953,11 @@ final public class PamModel implements PamSettings { // writeModuleList(); - return false; + return true; } /** - * Really just debu goutput to make a list of all modules ... + * Really just debug output to make a list of all modules ... */ private void writeModuleList() { ArrayList moduleInfoList = PamModuleInfo.getModuleList(); diff --git a/src/Spectrogram/SpectrogramDisplay.java b/src/Spectrogram/SpectrogramDisplay.java index dd2536d6..5c86913a 100644 --- a/src/Spectrogram/SpectrogramDisplay.java +++ b/src/Spectrogram/SpectrogramDisplay.java @@ -77,6 +77,7 @@ import pamScrollSystem.PamScrollerData; import pamScrollSystem.RangeSpinner; import pamScrollSystem.RangeSpinnerListener; import pamScrollSystem.jumping.ScrollJumper; +import pamguard.GlobalArguments; import soundPlayback.PlaybackControl; import soundPlayback.PlaybackProgressMonitor; import userDisplay.UserDisplayControl; @@ -274,7 +275,8 @@ InternalFrameListener, DisplayPanelContainer, SpectrogramParametersUser, PamSett // this should result in settings being loaded if they exist. PamSettingManager.getInstance().registerSettings(this); // always need to register, even if we're using old parameters // } - if (spectrogramParameters == null) { + boolean isBatch = GlobalArguments.getParam("-batch") != null; + if (spectrogramParameters == null && isBatch == false) { this.spectrogramParameters = new SpectrogramParameters(); PamView view = userDisplayControl.getPamView(); if (view != null) { @@ -285,6 +287,14 @@ InternalFrameListener, DisplayPanelContainer, SpectrogramParametersUser, PamSett } } } + if (spectrogramParameters == null) { + /* + * this can happen in batch mode if a display was added. + * Hopefully not a problem, but may need to set some parameters to + * set display up correctly. + */ + spectrogramParameters = new SpectrogramParameters(); + } spectrogramDisplay = this; diff --git a/src/binaryFileStorage/BinaryStore.java b/src/binaryFileStorage/BinaryStore.java index 4369c0f7..3db5fd66 100644 --- a/src/binaryFileStorage/BinaryStore.java +++ b/src/binaryFileStorage/BinaryStore.java @@ -155,6 +155,8 @@ PamSettingsSource, DataOutputStore { private PamControlledGUISwing binaryStoreGUISwing; private BackupInformation backupInformation; + + private BinaryDataMapMaker dataMapMaker; public static int getCurrentFileFormat() { return CURRENT_FORMAT; @@ -424,6 +426,25 @@ PamSettingsSource, DataOutputStore { } return true; } + + public boolean checkCommandLine() { + /* + * check to see if there is a command line override of the currently stored folder name. + */ + String globFolder = GlobalArguments.getParam(GlobalFolderArg); + if (globFolder == null) { + return false; + } + boolean ok = checkGlobFolder(globFolder); + if (ok) { + binaryStoreSettings.setStoreLocation(globFolder); // remember it. + return true; + } + else { + System.err.println("Unable to set binary storage folder " + globFolder); + return false; + } + } /** * Set and create if necessary the global folder. @@ -595,7 +616,8 @@ PamSettingsSource, DataOutputStore { // this first operation should be fast enough that it doesn't // need rethreading. - if (isViewer()) { + boolean hasCommandLine = checkCommandLine(); + if (isViewer() && !hasCommandLine) { BinaryStoreSettings newSettings = null; if (PamGUIManager.isSwing()) { //open the swing dialog. @@ -784,8 +806,8 @@ PamSettingsSource, DataOutputStore { * updates to the dialog to display progress, then close the * dialog. */ - BinaryDataMapMaker bdmm = new BinaryDataMapMaker(this); - AWTScheduler.getInstance().scheduleTask(bdmm); + dataMapMaker = new BinaryDataMapMaker(this); + AWTScheduler.getInstance().scheduleTask(dataMapMaker); } @@ -917,6 +939,7 @@ PamSettingsSource, DataOutputStore { } PamController.getInstance().notifyModelChanged(PamControllerInterface.CHANGED_OFFLINE_DATASTORE); // System.out.println("BinaryDataMapMaker really done " + this); + dataMapMaker = null; } @Override @@ -2547,5 +2570,18 @@ PamSettingsSource, DataOutputStore { BinaryStoreDeleter storeDeleter = new BinaryStoreDeleter(this); return storeDeleter.deleteDataFrom(timeMillis); } + @Override + public int getOfflineState() { + int state = super.getOfflineState(); + if (dataMapMaker != null) { + System.out.println("Binary store is map making"); + state = Math.max(state, PamController.PAM_MAPMAKING); + } + if (datagramManager != null & datagramManager.getStatus()) { + state = Math.max(state, PamController.PAM_MAPMAKING); + System.out.println("Binary store is creating datagram"); + } + return state; + } } diff --git a/src/dataGram/DatagramManager.java b/src/dataGram/DatagramManager.java index 1e8db795..5ae8aa66 100644 --- a/src/dataGram/DatagramManager.java +++ b/src/dataGram/DatagramManager.java @@ -12,6 +12,7 @@ import javax.swing.SwingWorker; import javafx.concurrent.Task; import pamViewFX.pamTask.PamTaskUpdate; import pamViewFX.pamTask.SimplePamTaskUpdate; +import pamguard.GlobalArguments; import dataMap.OfflineDataMap; import dataMap.OfflineDataMapPoint; import binaryFileStorage.BinaryOfflineDataMap; @@ -68,10 +69,11 @@ public class DatagramManager { public ArrayList checkAllDatagrams() { ArrayList grammedDataBlocks = getDataBlocks(); ArrayList updateBlocks = new ArrayList(); + boolean isBatch = GlobalArguments.getParam("-batch") != null; /** * Check to see if any of the blocks have an existing datagram */ - if (datagramSettings.validDatagramSettings == false) { + if (datagramSettings.validDatagramSettings == false && isBatch == false) { /* * this is the first time this has been run, so ask * the user what datagram size they want. @@ -169,6 +171,8 @@ public class DatagramManager { private volatile DatagramProgressDialog datagramProgressDialog; + private volatile DatagramCreator datagramCreator; + /** * update a list of datagrams. This should be done in a @@ -176,9 +180,17 @@ public class DatagramManager { * @param updateList */ public void updateDatagrams(ArrayList updateList) { - DatagramCreator datagramCreator = new DatagramCreator(updateList); + datagramCreator = new DatagramCreator(updateList); AWTScheduler.getInstance().scheduleTask(datagramCreator); } + + /** + * Get true if the datagram worker is running. + * @return + */ + public boolean getStatus() { + return datagramCreator != null; + } class DatagramCreatorTask extends Task { @@ -540,6 +552,7 @@ public class DatagramManager { PamController.getInstance().notifyTaskProgress( new SimplePamTaskUpdate("Finished Datagram Mapping", PamTaskUpdate.STATUS_DONE)); } + datagramCreator = null; } @Override diff --git a/src/generalDatabase/DBControlUnit.java b/src/generalDatabase/DBControlUnit.java index f3bc4b37..e3f9e625 100644 --- a/src/generalDatabase/DBControlUnit.java +++ b/src/generalDatabase/DBControlUnit.java @@ -50,6 +50,7 @@ public class DBControlUnit extends DBControl implements DataOutputStore { private boolean initialisationComplete; private BackupInformation backupInformation; + private CreateDataMap createDataMap; public DBControlUnit(String unitName) { this(null, unitName); @@ -188,8 +189,8 @@ public class DBControlUnit extends DBControl implements DataOutputStore { pamDataBlocks.get(i).removeOfflineDataMap(THIS); updateDataBlocks.add(pamDataBlocks.get(i)); } - - AWTScheduler.getInstance().scheduleTask(new CreateDataMap(updateDataBlocks)); + createDataMap = new CreateDataMap(updateDataBlocks); + AWTScheduler.getInstance().scheduleTask(createDataMap); } @@ -258,6 +259,7 @@ public class DBControlUnit extends DBControl implements DataOutputStore { */ public CreateDataMap(ArrayList loggingBlocks) { super(); + createDataMap = this; this.loggingBlocks = loggingBlocks; } @@ -379,6 +381,7 @@ public class DBControlUnit extends DBControl implements DataOutputStore { PamController.getInstance().notifyTaskProgress(new CreateMapInfo(PamTaskUpdate.STATUS_DONE)); PamController.getInstance().notifyModelChanged(PamControllerInterface.CHANGED_OFFLINE_DATASTORE); // System.out.println("Create datamap point: DONE2"); + createDataMap = null; } /* (non-Javadoc) @@ -508,5 +511,13 @@ public class DBControlUnit extends DBControl implements DataOutputStore { return getDbProcess().deleteDataFrom(timeMillis); } + @Override + public int getOfflineState() { + int state = super.getOfflineState(); + if (createDataMap != null) { + state = Math.max(state, PamController.PAM_MAPMAKING); + } + return state; + } } diff --git a/src/noiseMonitor/NoiseControl.java b/src/noiseMonitor/NoiseControl.java index 9a50ea6f..e1fbce42 100644 --- a/src/noiseMonitor/NoiseControl.java +++ b/src/noiseMonitor/NoiseControl.java @@ -13,6 +13,7 @@ import org.w3c.dom.Element; import PamController.PamControlledUnit; import PamController.PamControlledUnitSettings; +import PamController.PamController; import PamController.PamSettingManager; import PamController.PamSettings; import PamView.PamTabPanel; @@ -298,4 +299,15 @@ public class NoiseControl extends PamControlledUnit implements PamSettings { public NoiseSettings getNoiseSettings() { return noiseSettings; } + + @Override + public void notifyModelChanged(int changeType) { + super.notifyModelChanged(changeType); + switch (changeType) { + case PamController.INITIALIZATION_COMPLETE: + noiseProcess.setupProcess(); + sortBandEdges(); + } + } + } diff --git a/src/noiseMonitor/NoiseTabPanel.java b/src/noiseMonitor/NoiseTabPanel.java index 460b36ec..01140c31 100644 --- a/src/noiseMonitor/NoiseTabPanel.java +++ b/src/noiseMonitor/NoiseTabPanel.java @@ -789,7 +789,7 @@ private void setAxisLabels() { double[][] noiseData, lastNoisedata = null; int chan; int nMeasures = noiseDataBlock.getNumMeasurementBands(); - int x1, x2=0, y1, y2; + int x1, x2=0, y1=0, y2; int xWin = getWidth() / 10; synchronized (noiseDataBlock.getSynchLock()) { noiseIterator = noiseDataBlock.getListIterator(0); @@ -812,15 +812,22 @@ private void setAxisLabels() { lastNoisedata = lastUnit.getNoiseBandData(); } nMeasures = noiseData.length; + for (int i = 0; i < nMeasures; i++) { if (noiseDisplaySettings.isSelectData(i) == false) { continue; } + nStats = noiseData[i].length; for (int m = 0; m < nStats; m++) { if ((noiseDisplaySettings.selectedStats & noiseDataBlock.statIndexToBit(m)) == 0) { continue; } + try { y1 = dBToYPix(noiseData[i][m]+bandCorrection[i]); + } + catch (Exception e) { + e.printStackTrace(); + } setSymbol(symbol, chan, i, m); symbol.draw(g, new Point(x1, y1)); if (lastNoisedata != null && lastNoisedata.length > i) { diff --git a/src/offlineProcessing/OfflineTask.java b/src/offlineProcessing/OfflineTask.java index dab50a6b..ec63f39b 100644 --- a/src/offlineProcessing/OfflineTask.java +++ b/src/offlineProcessing/OfflineTask.java @@ -123,6 +123,16 @@ public abstract class OfflineTask { } } + /** + * Get a uniquely identifyng name for the task which consists of the + * pamControlledUnit type and name as well as the tasks shorter name from getName(); + * @return a long name which should be unique within a configuration. + */ + public String getLongName() { + PamControlledUnit tcu = getTaskControlledUnit(); + String str = String.format("%s:%s:%s", tcu.getUnitType(), tcu.getUnitName(), getName()); + return str; + } /** * * @return a name for the task, to be displayed in the dialog. diff --git a/src/offlineProcessing/OfflineTaskManager.java b/src/offlineProcessing/OfflineTaskManager.java index a88bfd1a..9c6485ce 100644 --- a/src/offlineProcessing/OfflineTaskManager.java +++ b/src/offlineProcessing/OfflineTaskManager.java @@ -17,6 +17,10 @@ public class OfflineTaskManager { private ArrayList globalTaskList = new ArrayList(); + public static final String commandFlag = "-offlinetask"; + + public ArrayList commandLineTasks = new ArrayList(); + public static OfflineTaskManager getManager() { if (singleInstance == null) { singleInstance = new OfflineTaskManager(); @@ -116,8 +120,9 @@ public class OfflineTaskManager { * @param offlineTask * @return matching task or null. */ + @SuppressWarnings("rawtypes") public OfflineTask findOfflineTask(OfflineTask offlineTask) { - return findOfflineTask(offlineTask.getUnitType(), offlineTask.getUnitName(), offlineTask.getName()); + return findOfflineTask(offlineTask.getLongName()); } /** @@ -128,6 +133,7 @@ public class OfflineTaskManager { * @param taskName * @return matching task or null. */ + @SuppressWarnings("rawtypes") public OfflineTask findOfflineTask(String unitType, String unitName, String taskName) { // could possibly also do a check on class type ???? for (OfflineTask aTask : globalTaskList) { @@ -143,4 +149,51 @@ public class OfflineTaskManager { } return null; } + + /** + * Another way of finding offline tasks based on their long name. This is basically + * the three names unitType, unitName and taskName concatenated together. Get's used + * for some task management such as passing batch processing instructions. + * @param taskLongName + * @return matching task or null. + */ + @SuppressWarnings("rawtypes") + public OfflineTask findOfflineTask(String taskLongName) { + for (OfflineTask aTask : globalTaskList) { + if (aTask.getLongName().equals(taskLongName)) { + return aTask; + } + } + return null; + } + + /** + * Add a task listed in the command line when PAMGuard was started. + * @param taskLongName + */ + public void addCommandLineTask(String taskLongName) { + commandLineTasks.add(taskLongName); + } + + /** + * The list of tasks from the command line. + * @return the commandLineTasks + */ + public ArrayList getCommandLineTasks() { + return commandLineTasks; + } + + /** + * Get the status of jobs to pass back to the batch process controller. + * @return + */ + public String getBatchStatus() { + /** + * this needs to largely follow the format of the data in folderinputsystem: + * String bs = String.format("%d,%d,%d,%s", nFiles,currentFile,generalStatus,currFile); + */ + int generalStatus = PamController.getInstance().getRealStatus(); + String bs = String.format("%d,%d,%d,%s", commandLineTasks.size(), 0, generalStatus, "Processing"); + return bs; + } } diff --git a/src/pamguard/GlobalArguments.java b/src/pamguard/GlobalArguments.java index 4f601911..eea82ec8 100644 --- a/src/pamguard/GlobalArguments.java +++ b/src/pamguard/GlobalArguments.java @@ -9,6 +9,11 @@ import java.util.HashMap; * */ public class GlobalArguments { + + /** + * Put some common flags here for convenience. + */ + public static final String BATCHFLAG = "-batch"; static HashMap globalFlags = new HashMap<>(); @@ -43,4 +48,12 @@ public class GlobalArguments { return Integer.valueOf(val); } + /** + * Is the batch flag set ? + * @return + */ + public static boolean isBatch() { + return getParam(BATCHFLAG) != null; + } + } diff --git a/src/pamguard/Pamguard.java b/src/pamguard/Pamguard.java index da9391ae..0a6da447 100644 --- a/src/pamguard/Pamguard.java +++ b/src/pamguard/Pamguard.java @@ -44,6 +44,7 @@ import binaryFileStorage.BinaryStore; import dataPlotsFX.JamieDev; import generalDatabase.DBControl; import networkTransfer.send.NetworkSender; +import offlineProcessing.OfflineTaskManager; import rocca.RoccaDev; import java.io.BufferedReader; @@ -176,6 +177,11 @@ public class Pamguard { else if (anArg.equalsIgnoreCase("-nosplash")) { showSplash = false; } + else if (anArg.equalsIgnoreCase(GlobalArguments.BATCHFLAG)) { + // flag to say we're in batch processing mode. Can be used + // to avoid one or two dialogs that pop up in Viewer mode. + GlobalArguments.setParam(GlobalArguments.BATCHFLAG, Boolean.TRUE.toString()); + } // removed SEICHE switch when the two SEICHE modules were converted to plugins // else if (anArg.equalsIgnoreCase("-seiche")) { @@ -244,6 +250,10 @@ public class Pamguard { pamBuoyGlobals.setMultiportConfig(mAddr, mPort); System.out.printf("Setting multicast control addr %s port %d\n", mAddr, mPort); } + else if (anArg.equalsIgnoreCase(OfflineTaskManager.commandFlag)) { + String taskName = args[iArg++]; + OfflineTaskManager.getManager().addCommandLineTask(taskName); + } else if (anArg.equalsIgnoreCase("-nolog")) { System.out.println("Disabling log file from command line switch..."); ProxyPrintStream.disableLogFile();