nordic-dev.net/PAMGuard
nordic-dev.net/PAMGuard
2
0
Fork 0
mirror of https://github.com/PAMGuard/PAMGuard.git synced 2024-11-21 22:52:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Squashed commit of the following:

Browse Source 
commit d5f504dcd1
Author: Douglas Gillespie <50671166+douggillespie@users.noreply.github.com>
Date:   Fri Nov 10 18:08:31 2023 +0000

    Bearing localiser offline save

    Bug stopping it correctly saving data from the offline task to the
    database fixed.

commit 7a44d49e27
Author: Douglas Gillespie <50671166+douggillespie@users.noreply.github.com>
Date:   Fri Oct 27 09:59:28 2023 +0100

    X3 Version 2.2.3

    Add maven jar files for X3/SUD version 2.2.3

commit fa5fe9943d
Author: Douglas Gillespie <50671166+douggillespie@users.noreply.github.com>
Date:   Thu Oct 26 14:44:41 2023 +0100

    update sud file management to more efficiently skip to the correct part
    of a sud file when reading data offline.

commit 60435e567a
Author: Brian S Miller <93690136+BrianSMiller@users.noreply.github.com>
Date:   Fri Sep 8 21:54:40 2023 +1000

    Fixes issues #111 and fixes #112 (DIFAR module crashes and ability to use Deep Learning Detections in DIFAR module) (#110)

    * Bugfix for OverlayMarks

    Check for a null pointer exception in OverlayMarks that was causing a crash on startup.

    * Bugfix for null pointer in symbol manager

    Fix a bug that I found where the DIFAR module was crashing the symbol manager. Seems that this was due to this subclass of clip generator having a null value for it's uniqueName. I've fixed by checking for null values and assigning a generic symbol when null.

    * DeepLearning detections canGenerateClips=true

    Set flag in DeepLearning detector so that detections are considered 'clips' by Pamguard. This allows them to be processed automatically in the DIFAR Localisation module (and maybe others).

    * DIFAR: bugfix frequency limits for auto-detections

    Fix a bug in DIFAR module where the frequency limits of automated detections were not being set properly by the DIFAR module.

    * DeepLearning - Bugfix to detection duration

    Fix bug in deep learning detector where duration (in samples) was being set to number of samples in a hop instead of the number of samples in a segment.
This commit is contained in:
Jamie Mac 2023-11-15 06:34:55 +00:00
parent 81907fedb8
commit 47662b5045
14 changed files with 165 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
repo/pamguard/org/x3/2.2.3/_remote.repositories Normal file
View File

@ -0,0 +1,4 @@
#NOTE: This is a Maven Resolver internal implementation file, its format can be changed without prior notice.
#Thu Oct 26 14:35:14 BST 2023
x3-2.2.3.jar>=
x3-2.2.3.pom>=

12
repo/pamguard/org/x3/2.2.3/m2e-lastUpdated.properties Normal file
View File

@ -0,0 +1,12 @@
#Thu Oct 26 15:50:39 BST 2023
bedatadriven|https\://nexus.bedatadriven.com/content/groups/public/|javadoc=1698327546071
repo|file\://C\:\\Users\\dg50\\source\\repos\\PAMGuardPAMGuard/repo|sources=1698327461721
repo|file\://C\:\\Users\\dg50\\source\\repos\\PAMGuardPAMGuard/repo|javadoc=1698327546071
talan|https\://nexus.talanlabs.com/content/repositories/releases/|javadoc=1698327546071
central|https\://repo1.maven.org/maven2|sources=1698331839689
unidata-all|https\://artifacts.unidata.ucar.edu/repository/unidata-all/|javadoc=1698327546071
talan|https\://nexus.talanlabs.com/content/repositories/releases/|sources=1698331839689
bedatadriven|https\://nexus.bedatadriven.com/content/groups/public/|sources=1698331839689
unidata-all|https\://artifacts.unidata.ucar.edu/repository/unidata-all/|sources=1698331839689
central|https\://repo1.maven.org/maven2|javadoc=1698327546071
repo|file\://C\:\\Users\\dg50\\source\\repos\\PAMGuardDG/repo|sources=1698331839689

12
repo/pamguard/org/x3/2.2.3/x3-2.2.3-javadoc.jar.lastUpdated Normal file
View File

@ -0,0 +1,12 @@
#NOTE: This is a Maven Resolver internal implementation file, its format can be changed without prior notice.
#Thu Oct 26 14:39:06 BST 2023
@default-talan-https\://nexus.talanlabs.com/content/repositories/releases/.lastUpdated=1698327545863
file\://C\:\\Users\\dg50\\source\\repos\\PAMGuardPAMGuard/repo/.lastUpdated=1698327544361
https\://repo1.maven.org/maven2/.error=
file\://C\:\\Users\\dg50\\source\\repos\\PAMGuardPAMGuard/repo/.error=
https\://artifacts.unidata.ucar.edu/repository/unidata-all/.error=
https\://nexus.bedatadriven.com/content/groups/public/.error=
https\://nexus.bedatadriven.com/content/groups/public/.lastUpdated=1698327545797
https\://artifacts.unidata.ucar.edu/repository/unidata-all/.lastUpdated=1698327545616
https\://repo1.maven.org/maven2/.lastUpdated=1698327546068
https\://nexus.talanlabs.com/content/repositories/releases/.error=Could not transfer artifact pamguard.org\:x3\:jar\:javadoc\:2.2.3 from/to talan (https\://nexus.talanlabs.com/content/repositories/releases/)\: nexus.talanlabs.com

14
repo/pamguard/org/x3/2.2.3/x3-2.2.3-sources.jar.lastUpdated Normal file
View File

@ -0,0 +1,14 @@
#NOTE: This is a Maven Resolver internal implementation file, its format can be changed without prior notice.
#Thu Oct 26 15:50:39 BST 2023
@default-talan-https\://nexus.talanlabs.com/content/repositories/releases/.lastUpdated=1698327461349
file\://C\:\\Users\\dg50\\source\\repos\\PAMGuardPAMGuard/repo/.lastUpdated=1698327459306
https\://repo1.maven.org/maven2/.error=
file\://C\:\\Users\\dg50\\source\\repos\\PAMGuardDG/repo/.error=
https\://nexus.bedatadriven.com/content/groups/public/.error=
https\://artifacts.unidata.ucar.edu/repository/unidata-all/.error=
https\://artifacts.unidata.ucar.edu/repository/unidata-all/.lastUpdated=1698327460817
https\://repo1.maven.org/maven2/.lastUpdated=1698327461652
https\://nexus.talanlabs.com/content/repositories/releases/.error=Could not transfer artifact pamguard.org\:x3\:jar\:sources\:2.2.3 from/to talan (https\://nexus.talanlabs.com/content/repositories/releases/)\: nexus.talanlabs.com
file\://C\:\\Users\\dg50\\source\\repos\\PAMGuardDG/repo/.lastUpdated=1698331839687
file\://C\:\\Users\\dg50\\source\\repos\\PAMGuardPAMGuard/repo/.error=
https\://nexus.bedatadriven.com/content/groups/public/.lastUpdated=1698327461288

BIN
repo/pamguard/org/x3/2.2.3/x3-2.2.3.jar Normal file
View File

Binary file not shown.

9
repo/pamguard/org/x3/2.2.3/x3-2.2.3.pom Normal file
View File

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<groupId>pamguard.org</groupId>
<artifactId>x3</artifactId>
<version>2.2.3</version>
<description>POM was created from install:install-file</description>
</project>

11
src/Acquisition/pamAudio/WavAudioFile.java
View File

@ -14,6 +14,7 @@ import javax.sound.sampled.UnsupportedAudioFileException;
import org.codehaus.plexus.util.FileUtils;
import Acquisition.offlineFuncs.AquisitionLoadPoint;
import PamDetection.RawDataUnit;
//import PamUtils.CPUMonitor;
import PamguardMVC.PamDataBlock;
import PamguardMVC.dataOffline.OfflineDataLoadInfo;
import dataMap.OfflineDataMap;
@ -70,9 +71,13 @@ public class WavAudioFile implements PamAudioFileLoader {
FileDataMapPoint mapPoint = offlineFileServer.findFirstMapPoint(mapIt, offlineDataLoadInfo.getStartMillis(), offlineDataLoadInfo.getEndMillis());
if (openSoundFile(mapPoint.getSoundFile()) == false) {
System.out.println("Could not open .wav sound file " + mapPoint.getSoundFile().getAbsolutePath());
System.out.println("Could not open sound file " + mapPoint.getSoundFile().getAbsolutePath());
return false;
}
if (offlineDataLoadInfo.cancel) {
return false;
}
File soundFile;
ByteConverter byteConverter = ByteConverter.createByteConverter(audioFormat);
@ -106,7 +111,11 @@ public class WavAudioFile implements PamAudioFileLoader {
try {
//System.out.println("Skipped " + skipped+ " " + skipBytes + " " + audioInputStream.available());
// CPUMonitor cpuMonitor = new CPUMonitor();
// cpuMonitor.start();
skipped = audioInputStream.skip(skipBytes);
// cpuMonitor.stop();
// System.out.println(cpuMonitor.getSummary("Sound skip: " + skipBytes + " bytes "));
//System.out.println("Offline " + (offlineDataLoadInfo.getStartMillis()-currentTime) + " ms : frame size: " + audioFormat.getFrameSize());
} catch (IOException e) {

2
src/PamView/paneloverlay/overlaymark/OverlayMarkProviders.java
View File

@ -49,7 +49,7 @@ public class OverlayMarkProviders {
continue;
}
String existingName = marker.getMarkerName();
if (existingName.equals(nameToAdd)) {
if (existingName!=null && existingName.equals(nameToAdd)) {
markProviders.remove(i);
break;
}

2
src/PamView/symbol/PamSymbolManager.java
View File

@ -125,7 +125,7 @@ abstract public class PamSymbolManager<T extends PamSymbolChooser> implements Pa
* This doesn't work since different displays will still need different options based on
* their projector - so it can work, but need to set the projector.
*/
if (managedSymbolData.useGeneric) {
if (managedSymbolData.useGeneric || displayName==null) {
displayName = GENERICNAME;
}

158
src/PamguardMVC/dataOffline/OfflineDataLoading.java
View File

@ -129,8 +129,8 @@ public class OfflineDataLoading<T extends PamDataUnit> {
this.orderOfflineData(offlineDataInfo);
}
/**
* Clears all raw and FFT data blocks prior to a new load.
*/
@ -140,7 +140,7 @@ public class OfflineDataLoading<T extends PamDataUnit> {
datablocks.get(i).clearAll();
datablocks.get(i).clearDeletedList();
}
datablocks = PamController.getInstance().getRawDataBlocks();
for (int i=0; i<datablocks.size(); i++) {
datablocks.get(i).clearAll();
@ -175,62 +175,62 @@ public class OfflineDataLoading<T extends PamDataUnit> {
long t4 = t2;
// String orderDates = String.format(" %s to %s",
// PamCalendar.formatDateTime(startMillis), PamCalendar.formatDateTime(endMillis));
// System.out.printf("Offline data order in %s %s from %s to %s\n", pamDataBlock.getDataName(), offlineDataInfo.toString(),
// PamCalendar.formatDBDateTime(offlineDataInfo.getStartMillis()),
// PamCalendar.formatDBDateTime(offlineDataInfo.getEndMillis()));
// if (offlineDataInfo.getEndMillis()-offlineDataInfo.getStartMillis() > 3600000L) {
// System.out.printf("Stupid long load time !");
// return;
// }
// System.out.printf("Offline data order in %s %s from %s to %s\n", pamDataBlock.getDataName(), offlineDataInfo.toString(),
// PamCalendar.formatDBDateTime(offlineDataInfo.getStartMillis()),
// PamCalendar.formatDBDateTime(offlineDataInfo.getEndMillis()));
// if (offlineDataInfo.getEndMillis()-offlineDataInfo.getStartMillis() > 3600000L) {
// System.out.printf("Stupid long load time !");
// return;
// }
try {
if (orderData != null) {
// System.out.println("order Data is not null");
if (orderData.isDone() == false) {
switch (offlineDataInfo.getInterrupt()) {
case OFFLINE_DATA_INTERRUPT:
// System.out.println("Request order cancelling");
if (orderData != null) {
// System.out.println("order Data is not null");
if (orderData.isDone() == false) {
switch (offlineDataInfo.getInterrupt()) {
case OFFLINE_DATA_INTERRUPT:
// System.out.println("Request order cancelling");
if (orderData.cancelOrder()) {
if (orderData.cancelOrder()) {
while (orderData!=null || !orderData.isDone()) {
try {
Thread.sleep(10);
} catch (InterruptedException e) {
e.printStackTrace();
return;
while (orderData!=null || !orderData.isDone()) {
try {
Thread.sleep(10);
} catch (InterruptedException e) {
e.printStackTrace();
return;
}
}
}
else {
// System.out.println("Old order could not be cancelled");
}
break;
case OFFLINE_DATA_CANCEL:
// System.out.println("Don't order new data " + orderDates);
return;
case OFFLINE_DATA_WAIT:
int waitCount = 0;
t3 = System.currentTimeMillis();
waitingDataLoads.add(offlineDataInfo);
return;
// // System.out.println("Wait for old lot to complete " + orderDates);
// while (true) {
// if (orderData==null || orderData.isDone() || orderData.isCancelled()) {
// break;
// }
// waitCount++;
// try {
// Thread.sleep(10, 0);
// } catch (InterruptedException e) {
// e.printStackTrace();
// }
// }
}
else {
// System.out.println("Old order could not be cancelled");
}
break;
case OFFLINE_DATA_CANCEL:
// System.out.println("Don't order new data " + orderDates);
return;
case OFFLINE_DATA_WAIT:
int waitCount = 0;
t3 = System.currentTimeMillis();
waitingDataLoads.add(offlineDataInfo);
return;
// // System.out.println("Wait for old lot to complete " + orderDates);
// while (true) {
// if (orderData==null || orderData.isDone() || orderData.isCancelled()) {
// break;
// }
// waitCount++;
// try {
// Thread.sleep(10, 0);
// } catch (InterruptedException e) {
// e.printStackTrace();
// }
// }
t4 = System.currentTimeMillis() - t3;
}
t4 = System.currentTimeMillis() - t3;
}
}
}
catch (NullPointerException e) {
// happens when orderData is set null in a different thread.
}
@ -244,8 +244,8 @@ public class OfflineDataLoading<T extends PamDataUnit> {
//t = new Timer(1000, new StartOrderOnTimer(orderData));
} // end of order lock
}
/**
* Cancels the current order.
*/
@ -259,16 +259,19 @@ public class OfflineDataLoading<T extends PamDataUnit> {
* @param que
*/
public void cancelDataOrder(boolean que) {
synchronized (orderLock) {
if (orderData != null) {
try {
orderData.cancelOrder();
}
catch (NullPointerException e) {
System.err.println("Null pointer in Cancel data order " + e.getMessage());
// threadMessage("Calling cancelDataOrder");
synchronized (orderLock) {
// threadMessage("cancelDataOrder is in synchronized oderLock");
if (orderData != null) {
try {
boolean isCancelled = orderData.cancelOrder();
// threadMessage("candelDataOrder returned: isCancelled = " + isCancelled);
}
catch (NullPointerException e) {
System.err.println("Null pointer in Cancel data order " + e.getMessage());
}
}
}
}
if (que) this.waitingDataLoads.clear();
}
/**
@ -319,16 +322,19 @@ public class OfflineDataLoading<T extends PamDataUnit> {
@Override
protected Integer doInBackground() {
// threadMessage("Start background");
try {
// System.out.println("Enter get offline data " + pamDataBlock.getDataName() + " Thread " + Thread.currentThread().getName());
// System.out.println("Enter get offline data " + pamDataBlock.getDataName() + " Thread " + Thread.currentThread().getName());
clearAllFFTBlocks();
// threadMessage("Called clearAllFTBlocks");
int ans = getOfflineData(offlineDataInfo);
// System.out.println("Leave get offline data " + pamDataBlock.getDataName());
// System.out.println("Leave get offline data " + pamDataBlock.getDataName());
if (this == orderData) {
orderData = null;
}
// threadMessage("End background exit code " + ans);
return ans;
}
catch (Exception e) {
@ -338,8 +344,8 @@ public class OfflineDataLoading<T extends PamDataUnit> {
}
return 0;
}
}
@Override
protected void done() {
@ -354,7 +360,7 @@ public class OfflineDataLoading<T extends PamDataUnit> {
if (isCancelled()) {
status = REQUEST_INTERRUPTED;
//check whether the cancelled thread should be saved for later loading.
// System.out.println("The load has been cancelled: " + offlineDataInfo.getCurrentObserver().getObserverName());
// System.out.println("The load has been cancelled: " + offlineDataInfo.getCurrentObserver().getObserverName());
checkDataReloadQue(offlineDataInfo);
}
else {
@ -376,11 +382,11 @@ public class OfflineDataLoading<T extends PamDataUnit> {
}
// System.out.println("WAITING DATA LOAD");
// System.out.println("WAITING DATA LOAD");
for (int i=0; i<waitingDataLoads.size(); i++){
// System.out.println("OfflineDataLoading.Done(): " + waitingDataLoads.get(i).getCurrentObserver().getObserverName());
// System.out.println("OfflineDataLoading.Done(): " + waitingDataLoads.get(i).getCurrentObserver().getObserverName());
}
// System.out.println("WAITING DATA END");
// System.out.println("WAITING DATA END");
if (!isCancelled()) launchQuedReloadThread();
@ -394,11 +400,17 @@ public class OfflineDataLoading<T extends PamDataUnit> {
}
// private void threadMessage(String message) {
// String name = Thread.currentThread().getName();
// String now = PamCalendar.formatDBDateTime(System.currentTimeMillis(), true);
// System.out.printf("Thread %s load for %s at %s: %s\n", name, pamDataBlock.getDataName(), now, message);
// }
private void launchQuedReloadThread(){
// System.out.println(" launchQuedReloadThread(): ");
// System.out.println(" launchQuedReloadThread(): ");
if (waitingDataLoads.size()>=1){
OfflineDataLoadInfo offlineDataInfo=waitingDataLoads.get(0);
// System.out.println("Removing: " + offlineDataInfo.getCurrentObserver().getObserverName());
// System.out.println("Removing: " + offlineDataInfo.getCurrentObserver().getObserverName());
waitingDataLoads.remove(0);
this.orderOfflineData(offlineDataInfo);
}
@ -470,15 +482,15 @@ public class OfflineDataLoading<T extends PamDataUnit> {
pamDataBlock.clearAll();
// System.out.println("Start loading some offline data: from " + PamCalendar.formatDateTime(offlineDataInfo.getStartMillis() ) +" to "+
// PamCalendar.formatDateTime(offlineDataInfo.getEndMillis() ) + " "+offlineDataInfo.getCurrentObserver().getObserverName() +
// " "+offlineDataInfo.getLoadKeepLayers());
// System.out.println("Start loading some offline data: from " + PamCalendar.formatDateTime(offlineDataInfo.getStartMillis() ) +" to "+
// PamCalendar.formatDateTime(offlineDataInfo.getEndMillis() ) + " "+offlineDataInfo.getCurrentObserver().getObserverName() +
// " "+offlineDataInfo.getLoadKeepLayers());
lastRequestAnswer = pamDataBlock.getParentProcess().getOfflineData(offlineDataInfo);
// System.out.println(String.format("getOfflineData %s has %d units ",
// getDataName(), getUnitsCount()));
// System.out.println("Orderring done: " + offlineDataInfo.getCurrentObserver().getObserverName() + " " + offlineDataInfo.getLoadKeepLayers());
// System.out.println("Orderring done: " + offlineDataInfo.getCurrentObserver().getObserverName() + " " + offlineDataInfo.getLoadKeepLayers());
//reset some of the changeable variables in offline data info.
offlineDataInfo.reset();

2
src/PamguardMVC/dataSelector/DataSelectorSettings.java
View File

@ -23,7 +23,9 @@ public class DataSelectorSettings implements Serializable, ManagedParameters {
if (selectorParams == null) {
selectorParams = new Hashtable<>();
}
if (name != null) {
selectorParams.put(name, params);
}
}
/**

1
src/bearinglocaliser/offline/BLOfflineTask.java
View File

@ -22,6 +22,7 @@ public class BLOfflineTask extends OfflineTask {
this.bearingLocaliserControl = bearingLocaliserControl;
bearingProcess = bearingLocaliserControl.getBearingProcess();
this.addRequiredDataBlock(rawOrFFTBlock = bearingProcess.getParentDataBlock());
addAffectedDataBlock(detectionBlock);
// PamDataBlock detectionSource = bearingLocaliserControl.getDetectionMonitor().getParentDataBlock();
// this.setParentDataBlock(detectionSource);
// setParentDataBlock(bearingProcess.getParentDataBlock());

14
src/difar/DifarProcess.java
View File

@ -316,11 +316,19 @@ public class DifarProcess extends PamProcess {
sP = difarControl.getDifarParameters().findSpeciesParams(difarDataUnit);
}
difarDataUnit.setDisplaySampleRate(sP.sampleRate);
if (!sP.useMarkedBandsForSpectrogramClips){
double[] frequency = {sP.processFreqMin, sP.processFreqMax};
difarDataUnit.setFrequency(frequency);
if (difarDataUnit.triggerName.equals(difarControl.getUnitName())) { // User detection
if (!sP.useMarkedBandsForSpectrogramClips ){
double[] frequency = {sP.processFreqMin, sP.processFreqMax};
difarDataUnit.setFrequency(frequency);
}
} else { //Check whether to override auto detection freq limits with DIFAR limits
if (!sP.useDetectionLimitsForTriggeredDetections ){ // Auto-detection
double[] frequency = {sP.processFreqMin, sP.processFreqMax};
difarDataUnit.setFrequency(frequency);
}
}
demuxDataUnit(difarDataUnit, demuxWorker, startTime);
calculateDifarGram(difarDataUnit, demuxWorker, startTime);

5
src/rawDeepLearningClassifier/dlClassification/DLClassifyProcess.java
View File

@ -106,7 +106,8 @@ public class DLClassifyProcess extends PamInstantProcess {
addOutputDataBlock(dlDetectionDataBlock);
dlDetectionDataBlock.setNaturalLifetimeMillis(600*1000); //keep this data for a while.
dlDetectionDataBlock.addDataAnnotationType(dlAnnotationType);
//ClipGeneration allows processing of detections by DIFAR module (and possibly others)
dlDetectionDataBlock.setCanClipGenerate(true);
//add custom graphics
PamDetectionOverlayGraphics overlayGraphics = new DLGraphics(dlModelResultDataBlock);
overlayGraphics.setDetectionData(true);
@ -482,7 +483,7 @@ public class DLClassifyProcess extends PamInstantProcess {
DataUnitBaseData basicData = groupDataBuffer.get(0).getBasicData().clone();
basicData.setMillisecondDuration(1000.*rawdata[0].length/this.sampleRate);
basicData.setSampleDuration((long) (groupDataBuffer.size()*dlControl.getDLParams().sampleHop));
basicData.setSampleDuration((long) (groupDataBuffer.size()*dlControl.getDLParams().rawSampleSize));
// System.out.println("Model result: " + modelResult.size());
DLDetection dlDetection = new DLDetection(basicData, rawdata);