nordic-dev.net/PAMGuard
nordic-dev.net/PAMGuard
2
0
Fork 0
mirror of https://github.com/PAMGuard/PAMGuard.git synced 2024-11-25 00:22:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Get CPOD click train import working.

Browse Source
This commit is contained in:
Jamie Mac 2024-02-14 14:27:11 +00:00
parent 45b0788c0d
commit 6314c90467
20 changed files with 414 additions and 222 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.classpath
View File

@ -6,8 +6,9 @@
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER">
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/jdk-21.0.2.13-hotspot">
<attributes>
<attribute name="module" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>

1
.settings/org.eclipse.core.resources.prefs
View File

@ -1,6 +1,5 @@
eclipse.preferences.version=1
encoding//src/rawDeepLearningClassifer/segmenter/SegmenterProcess.java=UTF-8
encoding//src/test=UTF-8
encoding//src/test/resources=UTF-8
encoding/<project>=UTF-8
encoding/src=UTF-8

6
.settings/org.eclipse.jdt.core.prefs
View File

@ -1,9 +1,9 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
org.eclipse.jdt.core.compiler.codegen.targetPlatform=21
org.eclipse.jdt.core.compiler.codegen.targetPlatform=18
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=21
org.eclipse.jdt.core.compiler.compliance=18
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
@ -13,4 +13,4 @@ org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning
org.eclipse.jdt.core.compiler.release=enabled
org.eclipse.jdt.core.compiler.source=21
org.eclipse.jdt.core.compiler.source=18

12
dependency-reduced-pom.xml
View File

@ -3,9 +3,9 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.pamguard</groupId>
<artifactId>Pamguard</artifactId>
<name>Pamguard Java12+</name>
<version>2.02.09d</version>
<description>Pamguard for Java 12+, using Maven to control dependcies</description>
<name>Pamguard</name>
<version>2.02.11a</version>
<description>Pamguard using Maven to control dependencies</description>
<url>www.pamguard.org</url>
<organization>
<name>Sea Mammal Research Unit, University of St. Andrews</name>
@ -13,6 +13,7 @@
</organization>
<build>
<sourceDirectory>${basedir}/src</sourceDirectory>
<testSourceDirectory>${basedir}/src/test</testSourceDirectory>
<resources>
<resource>
<directory>src</directory>
@ -62,7 +63,7 @@
</dependency>
</dependencies>
<configuration>
<release>17</release>
<release>21</release>
<compilerId>jdt</compilerId>
</configuration>
</plugin>
@ -188,6 +189,7 @@
<properties>
<maven.compiler.target>11</maven.compiler.target>
<maven.compiler.source>11</maven.compiler.source>
<javafx.version>16</javafx.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<javafx.version>21</javafx.version>
</properties>
</project>

2
pom.xml
View File

@ -4,7 +4,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.pamguard</groupId>
<artifactId>Pamguard</artifactId>
<version>2.02.09d</version>
<version>2.02.11a</version>
<name>Pamguard</name>
<description>Pamguard using Maven to control dependencies</description>
<url>www.pamguard.org</url>

1
src/clickDetector/ClickTrainDetector.java
View File

@ -58,7 +58,6 @@ public class ClickTrainDetector extends PamProcess implements PamSettings {
this.clickDataBlock = clickDataBlock;
PamSettingManager.getInstance().registerSettings(this);
clickDataBlock.addObserver(this);

58
src/cpod/CPODClick.java
View File

@ -27,8 +27,6 @@ public class CPODClick extends PamDataUnit<PamDataUnit,SuperDetection> implement
private short[] rawData;
/**
* The amplitude in dB.
*/
@ -38,6 +36,8 @@ public class CPODClick extends PamDataUnit<PamDataUnit,SuperDetection> implement
* The raw data transforms for the CPOD click
*/
private RawDataTransforms rawDataTransforms = null;
private CPODClassification cpodClassification;
/**
* Create a CPOD click. (This is used to load CPOD clicks from the binary store)
@ -147,7 +147,8 @@ public class CPODClick extends PamDataUnit<PamDataUnit,SuperDetection> implement
public static CPODClick makeCPODClick(long tMillis, long fileSamples, short[] shortData) {
short nCyc = shortData[3];
short bw = shortData[4];
short bw = shortData[4]; //bandwidth is an arbitary scale between 0 and 31;
bw = (short) ((255./31.) * (bw+1)); //make some attempt to convert to kHz
short kHz = shortData[5];
short endF = shortData[6];
short spl = shortData[7];
@ -326,6 +327,7 @@ public class CPODClick extends PamDataUnit<PamDataUnit,SuperDetection> implement
}
long tm = getTimeMilliseconds();
str += PamCalendar.formatDate(tm) + " " + PamCalendar.formatTime(tm, 3) + "<p>";
str += String.format("UID: %dkHz<p>", this.getUID());
str += String.format("Start Freq: %dkHz<p>", getkHz());
str += String.format("N Cycles: %d<p>", getnCyc());
str += String.format("BandWidth: %dkHz<p>", getBw());
@ -333,12 +335,12 @@ public class CPODClick extends PamDataUnit<PamDataUnit,SuperDetection> implement
str += String.format("Slope: %d<p>", getSlope());
str += String.format("SPL: %d", getSpl());
if (rawData.length == 40) {
str += String.format("<p>QClass %d, SpClass %d", getBits(rawData[19], (short) 0x3),
getBits(rawData[19], (short) 0b11100));
str += String.format("<p>QClass %d, SpClass %d", CPODUtils.getBits(rawData[19], (short) 0x3),
CPODUtils.getBits(rawData[19], (short) 0b11100));
str += String.format("<p>Train %d, %d click", rawData[20], rawData[23]);
str += String.format("<p>Qn %d, RateGood %d, SpGood %d, SpClass %d",
getBits(rawData[36], (short)3), getBits(rawData[36], (short) 4),
getBits(rawData[36], (short)8), getBits(rawData[36], (short) 240));
CPODUtils.getBits(rawData[36], (short)3), CPODUtils.getBits(rawData[36], (short) 4),
CPODUtils.getBits(rawData[36], (short)8), CPODUtils.getBits(rawData[36], (short) 240));
}
if (rawData != null) {
int nRaw = rawData.length;
@ -350,29 +352,18 @@ public class CPODClick extends PamDataUnit<PamDataUnit,SuperDetection> implement
}
}
}
if (this.getSuperDetection(0)!=null) {
str += String.format("I am part of a click train!!!: <p>");
CPODClickTrainDataUnit clicktrain = getCPODClickTrain();
if (clicktrain!=null) {
str += "<p>" + clicktrain.getStringInfo() + "<p>";
}
else {
str += String.format("I am not part of a click train :-(: <p>");
str += "<p>" + String.format("No click train info <p>");
}
// str += "<\html>";
return str;
}
short getBits(short data, short bitMap) {
short firstBit = 0;
for (int i = 0; i < 8; i++) {
if ((bitMap & (1<<i)) != 0) {
break;
}
firstBit++;
}
data &= bitMap;
return (short) (data>>firstBit);
}
@Override
public double[][] getWaveData() {
@ -405,5 +396,28 @@ public class CPODClick extends PamDataUnit<PamDataUnit,SuperDetection> implement
}
/**
* Get the click train detection from the click detection
* @return the click train detection
*/
public CPODClickTrainDataUnit getCPODClickTrain() {
for (int i=0; i<this.getSuperDetectionsCount(); i++) {
if (this.getSuperDetection(i) instanceof CPODClickTrainDataUnit) {
return (CPODClickTrainDataUnit) this.getSuperDetection(i);
}
}
return null;
}
public void setClassification(CPODClassification cpodClassification) {
this.cpodClassification = cpodClassification;
}
public CPODClassification getClassification() {
return this.cpodClassification;
}
}

6
src/cpod/CPODClickTrainDataBlock.java
View File

@ -62,7 +62,7 @@ public class CPODClickTrainDataBlock extends SuperDetDataBlock<CPODClickTrainDat
@Override
public void addPamData(CPODClickTrainDataUnit pamDataUnit) {
super.addPamData(pamDataUnit);
this.sortData();
//this.sortData();
}
/* (non-Javadoc)
@ -71,7 +71,7 @@ public class CPODClickTrainDataBlock extends SuperDetDataBlock<CPODClickTrainDat
@Override
public void addPamData(CPODClickTrainDataUnit pamDataUnit, Long uid) {
super.addPamData(pamDataUnit, uid);
this.sortData();
//this.sortData();
}
/* (non-Javadoc)
@ -80,7 +80,7 @@ public class CPODClickTrainDataBlock extends SuperDetDataBlock<CPODClickTrainDat
@Override
public void updatePamData(CPODClickTrainDataUnit pamDataUnit, long updateTimeMillis) {
super.updatePamData(pamDataUnit, updateTimeMillis);
this.sortData();
//this.sortData();
}
@Override

10
src/cpod/CPODClickTrainDataUnit.java
View File

@ -35,5 +35,15 @@ public class CPODClickTrainDataUnit extends DetectionGroupDataUnit implements Pa
public boolean isEcho() {
return cpodClassification.isEcho;
}
/**
* Gte information on the click train
* @return
*/
public String getStringInfo() {
return String.format("Species: %s Confidence %d is echo? %b", getSpecies(), getConfidence(), isEcho());
}
}

2
src/cpod/CPODControl2.java
View File

@ -144,8 +144,6 @@ public class CPODControl2 extends PamControlledUnit implements PamSettings {
CPODPlotProviderFX cpodPlotProviderFX = new CPODPlotProviderFX(this, cp1DataBlock);
TDDataProviderRegisterFX.getInstance().registerDataInfo(cpodPlotProviderFX);
// cpodPlotProviderFX = new CPODPlotProviderFX(this, cp3DataBlock);
TDDataProviderRegisterFX.getInstance().registerDataInfo(cpodPlotProviderFX);
// register the DD display
DDPlotRegister.getInstance().registerDataInfo(new CPODDPlotProvider(this, cp1DataBlock));
// DDPlotRegister.getInstance().registerDataInfo(new CPODDPlotProvider(this, cp3DataBlock));

192
src/cpod/CPODImporter.java
View File

@ -22,6 +22,7 @@ import binaryFileStorage.BinaryFooter;
import binaryFileStorage.BinaryObjectData;
import binaryFileStorage.BinaryOutputStream;
import binaryFileStorage.BinaryStore;
import cpod.CPODReader.CPODHeader;
import cpod.CPODUtils.CPODFileType;
import cpod.FPODReader.FPODHeader;
import cpod.FPODReader.FPODdata;
@ -39,7 +40,6 @@ public class CPODImporter {
public static final int MAX_SAVE = 2000000;
private File cpFile;
private long fileStart;
private CPODControl2 cpodControl;
@ -87,41 +87,6 @@ public class CPODImporter {
/**
* Import the CPOD file.
* @param cpFile2 - the cp1 file
* @param dataBlock - the datablock
* @param from - the click index to save from. e.g. 100 means that only click 100 + in the file is saved
* @param maxNum - the maximum number to import
* @return the total number of clicks in the file.
*/
private int importCPODFile(File cpFile2, CPODClickDataBlock dataBlock, CPODClickTrainDataBlock clickTrainDataBlock, int from, int maxNum) {
ArrayList<CPODClick> cpodData = null;
try {
cpodData = CPODReader.importCPODFile(cpFile, from, maxNum);
} catch (Exception e) {
e.printStackTrace();
return 0;
}
// fileStart + nMinutes * 60000L;
int nClicks = 0;
for (int i=0; i<cpodData.size(); i++) {
//System.out.println("Create a new CPOD click: ");
CPODClick cpodClick = processCPODClick(cpodData.get(i));
dataBlock.addPamData(cpodClick);
nClicks++;
}
cpodData=null; //trigger garbage collector if needed
return nClicks;
}
/**
* Process a CPOD click.
@ -264,7 +229,6 @@ public class CPODImporter {
class CPODImportTask extends Task<Integer> {
/**
* List of files, either CP1 or CP3
*/
@ -308,10 +272,7 @@ public class CPODImporter {
@Override
protected Integer call() throws Exception {
try {
System.out.println(("HELLO IMPORT THREAD: " ));
BinaryDataSource binarySource = cpodDataBlock.getBinaryDataSource();
binaryStore = (BinaryStore) PamController.getInstance().findControlledUnit(BinaryStore.defUnitType);
if (binaryStore == null) {
@ -332,7 +293,9 @@ public class CPODImporter {
if (this.isCancelled()) return -1;
final int ii = i;
this.updateMessage(("Importing CPOD file: " + (ii+1)));
System.out.println(("Importing CPOD file: " + (ii+1)) + " " +cpxFile.get(i));
this.updateMessage(("Importing CPOD file: " + (ii+1)) + " " +cpxFile.get(i).getName());
int nClicks = 0;
int totalClicks = Integer.MAX_VALUE;
@ -340,9 +303,6 @@ public class CPODImporter {
this.updateProgress(-1, 1);
CPODClickDataBlock tempCPODDataBlock = new CPODClickDataBlock("Temp CPOD Detections",
null, 0);
long[] fileStartEnd = getFileStartEnd(cpxFile.get(i));
while (importedClicks>0 || importedClicks==-1) {
@ -351,17 +311,20 @@ public class CPODImporter {
CPODFileType fileType = CPODUtils.getFileType(cpxFile.get(i));
System.out.println(("Importing from " + fileType + " " + i + " of " + cpxFile.size() + " from detection " + nClicks));
this.updateMessage(("Importing from " + fileType + " " + i + " of " + cpxFile.size() + " from detection " + nClicks));
//import the CPOD or FPOD data
this.updateProgress(-1, 1);
importedClicks = importFile(cpxFile.get(i), tempCPODDataBlock, clickTrainDataBlock, nClicks, MAX_SAVE);
importedClicks = importFile(cpxFile.get(i), cpodDataBlock, clickTrainDataBlock, nClicks, MAX_SAVE);
this.updateProgress(-1, 1);
System.out.println("Number of CPOD data units in the data block: " + tempCPODDataBlock.getUnitsCount() + " progress: " + (i+1) + " " + cpxFile.size() );
ListIterator<CPODClick> iterator = tempCPODDataBlock.getListIterator(0);
System.out.println("Number of CPOD data units in the data block: " + cpodDataBlock.getUnitsCount() + " progress: " + (i+1) + " " + cpxFile.size() );
//need to make a copy of the data incase we clear the cp2 datablock to look for previously
//loaded detection. Not memory efficient but the easiest way to do and only occurs
//in the laoding process.
ListIterator<CPODClick> iterator = cpodDataBlock.getDataCopy().listIterator();
cpodDataBlock.clearAll();
//save the click trains to the database
clickTrainDataBlock.saveViewerData();
@ -375,8 +338,6 @@ public class CPODImporter {
if (this.isCancelled()) return -1;
click = iterator.next();
System.out.println("Saving click: " + click.getUID());
//new binary file every daya;
cal.setTimeInMillis(click.getTimeMilliseconds());
int dayYear = cal.get(Calendar.DAY_OF_YEAR);
@ -397,7 +358,6 @@ public class CPODImporter {
data = cpodDataBlock.getBinaryDataSource().getPackedData(cpodTrainList.get(j));
this.binaryStream.storeData(data.getObjectType(), click.getBasicData(), data);
}
System.out.println("HAVE REMOVED " + count + " CP1 detections: " + cpodTrainList.size());
count = 0;
}
@ -425,11 +385,15 @@ public class CPODImporter {
//load the view from the data block for that day.
long millisDayStart = roundToDay(click.getTimeMilliseconds());
cpodDataBlock.loadViewerData(millisDayStart, millisDayStart+24*60*60*1000L-1, null);
System.out.println("Load viewer data from: " + PamCalendar.formatDBDateTime(millisDayStart) +
" to " +PamCalendar.formatDBDateTime(millisDayStart+24*60*60*1000L-1));
//now the tricky bit - if we are loading viewer files we want to save the data from the CP3 on top of the
//CP1 file but we DO NOT want to add in duplicate clicks.
if (cpodDataBlock.getUnitsCount()>0) {
System.out.println("There are already data units from CP1 or FP1 files " + cpodDataBlock.getUnitsCount());
cpodTrainList=null;
cpodTrainList = cpodDataBlock.getDataCopy();
Collections.sort(cpodTrainList, new CPODClickOmparator());
cpodDataBlock.clearAll(); //just incase
@ -463,10 +427,7 @@ public class CPODImporter {
this.binaryStream.storeData(data.getObjectType(), click.getBasicData(), data);
}
cpodDataBlock.clearAll();
tempCPODDataBlock.clearAll();
cpodDataBlock.clearAll();
//update number of clicks.
nClicks=nClicks+MAX_SAVE+1;
@ -484,29 +445,6 @@ public class CPODImporter {
return 1;
}
private long[] getFileStartEnd(File file) {
CPODFileType fileType = CPODUtils.getFileType(file);
long[] timelims = null;
switch (fileType){
case CP1:
case CP3:
//TODO
break;
case FP1:
case FP3:
FPODHeader header =FPODReader.readHeader(file);
if (header==null) return null;
timelims=new long[] {CPODUtils.podTimeToMillis(header.FirstLoggedMin), CPODUtils.podTimeToMillis(header.LastLoggedMin)};
break;
default:
break;
}
return timelims;
}
/**
@ -524,6 +462,68 @@ public class CPODImporter {
calendar.set(Calendar.MILLISECOND, 0);
return calendar.getTimeInMillis();
}
/**
* Import the CPOD file.
* @param cpFile2 - the cp1 file
* @param dataBlock - the datablock
* @param from - the click index to save from. e.g. 100 means that only click 100 + in the file is saved
* @param maxNum - the maximum number to import
* @return the total number of clicks in the file.
*/
private int importCPODFile(File cpFile2, CPODClickDataBlock dataBlock, CPODClickTrainDataBlock clickTrainDataBlock, int from, int maxNum) {
ArrayList<CPODClick> cpodData = null;
try {
cpodData = CPODReader.importCPODFile(cpFile2, from, maxNum);
} catch (Exception e) {
e.printStackTrace();
return 0;
}
HashMap<Integer, CPODClickTrainDataUnit> cpodClickTrains = new HashMap<Integer, CPODClickTrainDataUnit>();
// fileStart + nMinutes * 60000L;
int nClicks = 0;
for (int i=0; i<cpodData.size(); i++) {
//System.out.println("Create a new CPOD click: ");
CPODClick cpodClick = processCPODClick(cpodData.get(i));
dataBlock.addPamData(cpodClick);
if (cpodData.get(i).getClassification()!=null) {
CPODClickTrainDataUnit clickTrain = cpodClickTrains.get(cpodData.get(i).getClassification().clicktrainID);
//add the click train to the hash map
if (clickTrain==null) {
clickTrain= new CPODClickTrainDataUnit(cpodClick.getTimeMilliseconds(), null, cpodData.get(i).getClassification());
cpodClickTrains.put(cpodData.get(i).getClassification().clicktrainID, clickTrain);
}
//add the cpos click as a sub detection
clickTrain.addSubDetection(cpodClick);
}
nClicks++;
}
//add all the click trains with sub detections ot the datablock.
int count =0;
for (Integer key: cpodClickTrains.keySet()) {
if (count%100 ==0) {
this.updateMessage(("Add click train data to datablock: " + count + " of " + cpodClickTrains.keySet().size()));
this.updateProgress(count, cpodClickTrains.keySet().size());
// System.out.println("Add click train data to datablock: " + count + " " + cpodClickTrains.keySet().size());
}
clickTrainDataBlock.addPamData(cpodClickTrains.get(key));
count++;
}
cpodData=null; //trigger garbage collector if needed
return nClicks;
}
/**
@ -534,6 +534,11 @@ public class CPODImporter {
* @return the total number of clicks in the file.
*/
protected int importFPODFile(File cpFile, CPODClickDataBlock dataBlock, CPODClickTrainDataBlock clickTrainDataBlock, int from, int maxNum) {
/**
* Note that this function is very simialr to the CPOD ijport v- we could abstract it but, ther eis likely to be a few changes to these fucntions a
* as FPOD data is further developed and so making this enat may be far more trouble than it worth.
*/
ArrayList<FPODdata> fpodData = new ArrayList<FPODdata>();
@ -613,8 +618,33 @@ public class CPODImporter {
}
return 0;
}
/**
* Get the start and end of a file from the header data. This opens and closes the file.
* @param file -a CP1, CP3 FP1 or FP3 file;
* @return the start and end time in Java millis in a long array (first element start, last element end)
*/
private long[] getFileStartEnd(File file) {
CPODFileType fileType = CPODUtils.getFileType(file);
long[] timelims = null;
switch (fileType){
case CP1:
case CP3:
CPODHeader headerC =CPODReader.readHeader(file);
timelims=new long[] {headerC.fileStart, headerC.fileEnd};
break;
case FP1:
case FP3:
FPODHeader header =FPODReader.readHeader(file);
if (header==null) return null;
timelims=new long[] {CPODUtils.podTimeToMillis(header.FirstLoggedMin), CPODUtils.podTimeToMillis(header.LastLoggedMin)};
break;
default:
break;
}
return timelims;
}
/**
* Import a file.

146
src/cpod/CPODReader.java
View File

@ -8,8 +8,11 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import PamUtils.PamCalendar;
import cpod.CPODUtils.CPODFileType;
import cpod.FPODReader.FPODdata;
/**
* Read CPOD data.
@ -21,8 +24,8 @@ import cpod.CPODUtils.CPODFileType;
*
*/
public class CPODReader {
/**
* A new minute. Don;t think we need to do anything here.?
* @param byteData
@ -32,7 +35,24 @@ public class CPODReader {
}
public static CPODHeader readHeader(File cpFile) {
BufferedInputStream bis = null;
FileInputStream fileInputStream = null;
try {
bis = new BufferedInputStream(fileInputStream = new FileInputStream(cpFile));
} catch (FileNotFoundException e) {
e.printStackTrace();
return null;
}
return readHeader( bis, CPODUtils.getFileType(cpFile));
}
public static CPODHeader readHeader(BufferedInputStream bis, CPODFileType cpFileType) {
int bytesRead;
byte[] headData = new byte[getHeadSize(cpFileType)];
@ -53,7 +73,7 @@ public class CPODReader {
try {
shortData[i] = dis.readShort();
if (shortData[i] == 414) {
// System.out.println("Found id at %d" + i);
// System.out.println("Found id at %d" + i);
}
} catch (IOException e) {
e.printStackTrace();
@ -79,19 +99,19 @@ public class CPODReader {
intData[i] = dis.readInt();
int bOff = i*4;
int sOff = i*2;
// if (intData[i] > 0)
// System.out.println(String.format("%d, Int = %d, Float = %3.5f, Short = %d,%d, bytes = %d,%d,%d,%d", i, intData[i],
// floatData[i],
// shortData[sOff], shortData[sOff+1],
// headData[bOff], headData[bOff+1], headData[bOff+2], headData[bOff+3]));
// if (intData[i] > 0)
// System.out.println(String.format("%d, Int = %d, Float = %3.5f, Short = %d,%d, bytes = %d,%d,%d,%d", i, intData[i],
// floatData[i],
// shortData[sOff], shortData[sOff+1],
// headData[bOff], headData[bOff+1], headData[bOff+2], headData[bOff+3]));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
CPODHeader header = new CPODHeader();
header.fileStart = CPODUtils.podTimeToMillis(intData[64]);
header.fileEnd = CPODUtils.podTimeToMillis(intData[65]);
// other times seem to be packed in ints 66 - 69.
@ -100,7 +120,7 @@ public class CPODReader {
return header;
}
/**
* Import a CPOD file.
@ -110,22 +130,27 @@ public class CPODReader {
* @return the total number of clicks in the file.
*/
protected static ArrayList<CPODClick> importCPODFile(File cpFile, int from, int maxNum) {
//holds a map of the click train detections.
HashMap<Integer, CPODClassification> clickTrains = new HashMap<Integer, CPODClassification>();
//the current classification
CPODClassification cpodClassification;
ArrayList<CPODClick> clicks = new ArrayList<CPODClick>();
BufferedInputStream bis = null;
int bytesRead;
FileInputStream fileInputStream = null;
long totalBytes = 0;
CPODFileType cpFileType = CPODUtils.getFileType(cpFile);
try {
bis = new BufferedInputStream(fileInputStream = new FileInputStream(cpFile));
} catch (FileNotFoundException e) {
e.printStackTrace();
return null;
}
CPODHeader header = readHeader(bis, cpFileType);
if (header == null) {
return null;
@ -158,24 +183,67 @@ public class CPODReader {
isClick = byteData[dataSize-1] != -2;
if (isClick) {
nClicks++;
if (from<0 || (nClicks>from && nClicks<(from+maxNum))) {
//System.out.println("Create a new CPOD click: ");
CPODClick cpodClick = processCPODClick(nMinutes, shortData, header);
if (cpFileType.equals(CPODFileType.CP3)) {
short trainID = shortData[20];
short species = (short) (shortData[36] & (112 >> 4));
short quality = (short) (shortData[36] & 3);
// short species = (short) ((shortData[19] >> 2) & 7);
//
// short quality = (short) (shortData[19] & 3);
//generate a unique train ID within the file
int trainUID = Integer.valueOf(String.format("%06d", nMinutes) + String.format("%d", trainID));
//find the click train from the hash map - if it is not there, create a new one.
cpodClassification = clickTrains.get(trainUID);
if (cpodClassification==null) {
cpodClassification = new CPODClassification();
cpodClassification.isEcho = false;
cpodClassification.clicktrainID = trainUID;
cpodClassification.species = CPODUtils.getSpecies(species);
cpodClassification.qualitylevel = quality;
clickTrains.put(trainUID, cpodClassification);
System.out.println("Click train ID: " + trainUID + " minutes: " + nMinutes + " species: " + species + " quality level: " + quality);
}
cpodClick.setClassification(cpodClassification);
}
clicks.add(cpodClick);
if (nClicks%100000==0) {
System.out.println("CPOD data: " + nClicks + " " + PamCalendar.formatDateTime(cpodClick.getTimeMilliseconds()) + " " +cpodClick.getBw() + " " +shortData[4] );
// if (nClicks>0)return clicks; //FIXME
}
}
// // now remove the data unit from the data block in order to clear up memory. Note that the remove method
nClicks++;
// // now remove the data unit from the data block in order to clear up memory. Note that the remove method
// // saves the data unit to the Deleted-Items list, so clear that as well (otherwise we'll just be using
// // up all the memory with that one)
// dataBlock.remove(cpodClick);
// dataBlock.clearDeletedList();
}
else {
nMinutes ++;
@ -191,7 +259,7 @@ public class CPODReader {
return clicks;
}
public static int getHeadSize(CPODFileType fileType) {
switch (fileType) {
case CP1:
@ -219,7 +287,7 @@ public class CPODReader {
}
return 0;
}
/**
* Holds an CPOD header information
* <p>
@ -231,9 +299,9 @@ public class CPODReader {
public short podId;
public long fileEnd;
public long fileStart;
}
/**
* Is it the end of the file ?
* @param byteData
@ -250,7 +318,7 @@ public class CPODReader {
}
return true;
}
/**
* Create a CPOD click object from CPOD data.
* @param nMinutes
@ -259,25 +327,41 @@ public class CPODReader {
* @return
*/
private static CPODClick processCPODClick(int nMinutes, short[] shortData, CPODHeader header) {
long minuteMillis = header.fileStart + nMinutes * 60000L;
int t = shortData[0]<<16 |
shortData[1]<<8 |
shortData[2]; // 5 microsec intervals !
long tMillis = minuteMillis + t/200;
// do a sample number within the file as 5us intervals
long fileSamples = t + minuteMillis * 200;
/*
*
*/
return CPODClick.makeCPODClick(tMillis, fileSamples, shortData);
}
/**
* Test the program
* @param args
*/
public static void main(String[] args) {
// String filePath = "/Users/au671271/Library/CloudStorage/GoogleDrive-macster110@gmail.com/My Drive/PAMGuard_dev/CPOD/FPOD_NunBank/0866 NunBankB 2023 06 27 FPOD_6480 file0.FP1";
String filePath = "D:\\Dropbox\\PAMGuard_dev\\tutorials\\CPOD_wav\\data\\Hyskeir\\CPOD\\0740 Hyskeir 2022 12 02 POD1655 file01.CP3";
// String filePath = "D:\\DropBox\\PAMGuard_dev\\CPOD\\FPOD_NunBank\\0866 NunBankB 2023 06 27 FPOD_6480 file0.FP3";
File fpfile = new File(filePath);
importCPODFile( fpfile, 0, Integer.MAX_VALUE);
}
}

62
src/cpod/CPODSpeciesModifier.java Normal file
View File

@ -0,0 +1,62 @@
package cpod;
import java.awt.Color;
import PamView.GeneralProjector;
import PamView.PamSymbolType;
import PamView.symbol.PamSymbolChooser;
import PamView.symbol.SymbolData;
import PamView.symbol.modifier.SymbolModType;
import PamView.symbol.modifier.SymbolModifier;
import PamguardMVC.PamDataUnit;
import cpod.CPODClassification.CPODSpeciesType;
public class CPODSpeciesModifier extends SymbolModifier {
final Color porpColor = new Color(93,30,255);
final Color dolphColor = new Color(255,160,0);
private SymbolData symbolData = new SymbolData(PamSymbolType.SYMBOL_CIRCLE, 5, 5, true, java.awt.Color.BLACK, java.awt.Color.BLACK);
public CPODSpeciesModifier(PamSymbolChooser symbolChooser) {
super("Species", symbolChooser, SymbolModType.FILLCOLOUR | SymbolModType.LINECOLOUR);
}
@Override
public SymbolData getSymbolData(GeneralProjector projector, PamDataUnit dataUnit) {
CPODClick cpodClick = (CPODClick) dataUnit;
Color color = Color.BLACK;
if (cpodClick.getCPODClickTrain()!=null) {
CPODSpeciesType species = cpodClick.getCPODClickTrain().getSpecies();
switch(species) {
case DOLPHIN:
color=dolphColor;
break;
case NBHF:
color=porpColor;;
break;
case SONAR:
color = Color.DARK_GRAY;
break;
case UNKNOWN:
break;
default:
break;
}
}
symbolData.setFillColor(color);
symbolData.setLineColor(color);
return symbolData;
}
}

4
src/cpod/CPODSymbolManager.java
View File

@ -51,9 +51,9 @@ public class CPODSymbolManager extends StandardSymbolManager {
psc.addSymbolModifier(new PeakFreqModifier(psc));
//add the peak frequency modifier that allows clicks to be coloured by peak frequency.
psc.addSymbolModifier(new CPODClickTrainModifier(psc));
psc.addSymbolModifier(new CPODSpeciesModifier(psc));
// we can also add some default behaviour here to match the old behaviour
// we can also add some default behaviour here to match the old behaviourzz
// these will get overridden once user options are set, but it's good to give defaults.
// SymbolModifier eventMod = psc.hasSymbolModifier(SuperDetSymbolModifier.class);
// if (eventMod != null) {

2
src/cpod/CPODTrainSymbolManager.java
View File

@ -7,7 +7,7 @@ public class CPODTrainSymbolManager extends SuperDetectionSymbolManager {
public CPODTrainSymbolManager(CPODClickTrainDataBlock pamDataBlock) {
super(pamDataBlock, new SymbolData());
super.setSpecialColourName("Event Colour");
super.setSpecialColourName("Click Train");
}

12
src/cpod/CPODUtils.java
View File

@ -111,5 +111,17 @@ public class CPODUtils {
public static CPODSpeciesType getSpecies(String species) {
return CPODSpeciesType.valueOf(species);
}
public static short getBits(short data, short bitMap) {
short firstBit = 0;
for (int i = 0; i < 8; i++) {
if ((bitMap & (1<<i)) != 0) {
break;
}
firstBit++;
}
data &= bitMap;
return (short) (data>>firstBit);
}
}

1
src/cpod/FPODReader.java
View File

@ -419,6 +419,7 @@ public class FPODReader {
cpodClassification.isEcho = echo;
cpodClassification.clicktrainID = trainUID;
cpodClassification.species = CPODUtils.getSpecies(species);
cpodClassification.qualitylevel = qualitylevel;
clickTrains.put(trainUID, cpodClassification);

1
src/cpod/dataSelector/StandardCPODataFilter.java
View File

@ -44,6 +44,7 @@ public class StandardCPODataFilter implements CPODDataFilter {
return 1;
}
else {
System.out.println("Do not plot data: " +cpodClick.getTimeMilliseconds());
return 0;
}

3
src/cpod/fx/CPODSettingsPane.java
View File

@ -352,7 +352,6 @@ public class CPODSettingsPane extends SettingsPane<CPODParams> {
* Called whenever the importing is finished.
*/
private void importingFinished() {
System.out.println("IMPORT FINISHED: MEH~!");
if (PamGUIManager.isSwing())
//important to put this here or , if a swing dialog pop ups, then there is thread lock.
@ -450,7 +449,7 @@ public class CPODSettingsPane extends SettingsPane<CPODParams> {
@Override
public CPODParams getParams(CPODParams currParams) {
currParams.offlineFolder = currentFolder.getAbsolutePath();
currParams.offlineFolder = currentFolder==null? null:currentFolder.getAbsolutePath();
currParams.subFolders = subFolder.selectedProperty().get();
try {
currParams.startOffset = startOffset.getValue();

112
src/decimator/layoutFX/DecimatorSettingsPane.java
View File

@ -1,39 +1,17 @@
package decimator.layoutFX;
import java.awt.BorderLayout;
import java.awt.GridBagConstraints;
import java.awt.GridBagLayout;
import javax.swing.AbstractButton;
import javax.swing.BoxLayout;
import javax.swing.JButton;
import javax.swing.JComboBox;
import javax.swing.JLabel;
import javax.swing.JPanel;
import javax.swing.JTabbedPane;
import javax.swing.JTextField;
import javax.swing.border.TitledBorder;
import PamController.PamController;
import PamController.SettingsPane;
import PamDetection.RawDataUnit;
import PamView.dialog.PamGridBagContraints;
import PamView.dialog.SourcePanel;
import PamView.panel.PamAlignmentPanel;
import PamguardMVC.PamDataBlock;
import dataMap.filemaps.OfflineFileDialogPanel;
import decimator.DecimatorControl;
import decimator.DecimatorParams;
import decimator.DecimatorParamsDialog.DefaultFilterButton;
import decimator.DecimatorParamsDialog.FilterButton;
import decimator.DecimatorParamsDialog.SPMonitor;
import decimator.DecimatorParamsDialog.SPSelection;
import javafx.scene.Node;
import javafx.scene.control.Label;
import javafx.scene.control.TextField;
import javafx.scene.layout.Pane;
import pamViewFX.fxNodes.PamBorderPane;
import pamViewFX.fxNodes.PamButton;
import pamViewFX.fxNodes.PamGridPane;
import pamViewFX.fxNodes.PamVBox;
import pamViewFX.fxNodes.utilityPanes.SourcePaneFX;
@ -95,51 +73,53 @@ public class DecimatorSettingsPane extends SettingsPane<DecimatorParams> {
decimatorPanel.add(sourceSampleRate = new Label(" - Hz"), gridx, gridy);
gridx = 0;
gridy ++;
decimatorPanel.add(new JLabel("Output sample rate "), gridx, gridy);
gridx ++;
decimatorPanel.add(newSampleRate = new TextField(), gridx, gridy);
gridx ++;
decimatorPanel.add(new JLabel(" Hz"), gridx, gridy);
gridy ++;
gridx = 0;
gridwidth = 1;
decimatorPanel.add(filterButton = new PamButton("Filter settings"), gridx, gridy);
filterButton.addActionListener(new FilterButton());
gridx = 1;
gridwidth = 2;
addComponent(decimatorPanel, defaultFilterButton = new PamButton("Default Filter"), gridx, gridy);
defaultFilterButton.addActionListener(new DefaultFilterButton());
gridx = 0;
gridwidth = 3;
gridy++;
addComponent(decimatorPanel, filterInfo = new JLabel("Filter: "), gridx, gridy);
gridx = 0;
gridwidth = 1;
gridy++;
addComponent(decimatorPanel, new JLabel("Interpolation: ", JLabel.RIGHT), gridx, gridy);
gridx += gridwidth;
gridwidth = 2;
addComponent(decimatorPanel, interpolator = new JComboBox<String>(), gridx, gridy);
interpolator.addItem("None");
interpolator.addItem("Linear");
interpolator.addItem("Quadratic");
isViewer = PamController.getInstance().getRunMode() == PamController.RUN_PAMVIEW;
if (isViewer) {
JTabbedPane tabbedPane = new JTabbedPane();
offlineDAQDialogPanel = new OfflineFileDialogPanel(decimatorControl, this);
tabbedPane.add("Offline Files", offlineDAQDialogPanel.getComponent());
tabbedPane.add("Runtime Settings", mainPanel);
setDialogComponent(tabbedPane);
}
else {
setDialogComponent(mainPanel);
}
// decimatorPanel.add(new JLabel("Output sample rate "), gridx, gridy);
// gridx ++;
// decimatorPanel.add(newSampleRate = new TextField(), gridx, gridy);
// gridx ++;
// decimatorPanel.add(new JLabel(" Hz"), gridx, gridy);
// gridy ++;
// gridx = 0;
// gridwidth = 1;
// decimatorPanel.add(filterButton = new PamButton("Filter settings"), gridx, gridy);
// filterButton.addActionListener(new FilterButton());
// gridx = 1;
// gridwidth = 2;
// addComponent(decimatorPanel, defaultFilterButton = new PamButton("Default Filter"), gridx, gridy);
// defaultFilterButton.addActionListener(new DefaultFilterButton());
// gridx = 0;
// gridwidth = 3;
// gridy++;
// addComponent(decimatorPanel, filterInfo = new JLabel("Filter: "), gridx, gridy);
// gridx = 0;
// gridwidth = 1;
// gridy++;
// addComponent(decimatorPanel, new JLabel("Interpolation: ", JLabel.RIGHT), gridx, gridy);
// gridx += gridwidth;
// gridwidth = 2;
// addComponent(decimatorPanel, interpolator = new JComboBox<String>(), gridx, gridy);
// interpolator.addItem("None");
// interpolator.addItem("Linear");
// interpolator.addItem("Quadratic");
//
// isViewer = PamController.getInstance().getRunMode() == PamController.RUN_PAMVIEW;
// if (isViewer) {
// JTabbedPane tabbedPane = new JTabbedPane();
// offlineDAQDialogPanel = new OfflineFileDialogPanel(decimatorControl, this);
// tabbedPane.add("Offline Files", offlineDAQDialogPanel.getComponent());
// tabbedPane.add("Runtime Settings", mainPanel);
// setDialogComponent(tabbedPane);
// }
// else {
// setDialogComponent(mainPanel);
// }
//
// setHelpPoint("sound_processing.decimatorHelp.docs.decimator_decimator");
// filterButton.setToolTipText("Manual adjustment of filter settings");
// defaultFilterButton.setToolTipText("Set a default filter (6th order Butterworth low pass at Decimator Nyquist frequency)");
// interpolator.setToolTipText("If Decimation / upsampling is not by an integer value, you should use interpolation to improve waveform reconstruction");
setHelpPoint("sound_processing.decimatorHelp.docs.decimator_decimator");
filterButton.setToolTipText("Manual adjustment of filter settings");
defaultFilterButton.setToolTipText("Set a default filter (6th order Butterworth low pass at Decimator Nyquist frequency)");
interpolator.setToolTipText("If Decimation / upsampling is not by an integer value, you should use interpolation to improve waveform reconstruction");
return mainPanel;
}