diff --git a/src/cpod/CPODImporter.java b/src/cpod/CPODImporter.java index d6520b68..1198b9dd 100644 --- a/src/cpod/CPODImporter.java +++ b/src/cpod/CPODImporter.java @@ -3,6 +3,8 @@ package cpod; import java.io.File; import java.util.ArrayList; import java.util.Calendar; +import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -21,6 +23,7 @@ import binaryFileStorage.BinaryObjectData; import binaryFileStorage.BinaryOutputStream; import binaryFileStorage.BinaryStore; import cpod.CPODUtils.CPODFileType; +import cpod.FPODReader.FPODHeader; import cpod.FPODReader.FPODdata; import javafx.application.Platform; import javafx.concurrent.Task; @@ -62,7 +65,7 @@ public class CPODImporter { return t ; }); - + public CPODImporter(CPODControl2 cpodControl) { @@ -81,8 +84,8 @@ public class CPODImporter { // dataBlock = cpodControl.getCP3DataBlock(); // // } - - + + /** * Import the CPOD file. @@ -94,18 +97,18 @@ public class CPODImporter { */ private int importCPODFile(File cpFile2, CPODClickDataBlock dataBlock, CPODClickTrainDataBlock clickTrainDataBlock, int from, int maxNum) { ArrayList cpodData = null; - + try { cpodData = CPODReader.importCPODFile(cpFile, from, maxNum); } catch (Exception e) { e.printStackTrace(); return 0; } - - -// fileStart + nMinutes * 60000L; - - + + + // fileStart + nMinutes * 60000L; + + int nClicks = 0; for (int i=0; i> importCPODData(List files) { - + List> tasks = new ArrayList>(); for (int i=0; i cpodTrainList; /** * @@ -343,6 +309,9 @@ public class CPODImporter { protected Integer call() throws Exception { try { + System.out.println(("HELLO IMPORT THREAD: " )); + + BinaryDataSource binarySource = cpodDataBlock.getBinaryDataSource(); binaryStore = (BinaryStore) PamController.getInstance().findControlledUnit(BinaryStore.defUnitType); if (binaryStore == null) { @@ -370,27 +339,32 @@ public class CPODImporter { int importedClicks = -1; this.updateProgress(-1, 1); - + CPODClickDataBlock tempCPODDataBlock = new CPODClickDataBlock("Temp CPOD Detections", null, 0); + long[] fileStartEnd = getFileStartEnd(cpxFile.get(i)); + while (importedClicks>0 || importedClicks==-1) { - + CPODFileType fileType = CPODUtils.getFileType(cpxFile.get(i)); - - this.updateMessage(("Importing from " + fileType + " " + i + " of " + cpxFile.size())); + + System.out.println(("Importing from " + fileType + " " + i + " of " + cpxFile.size() + " from detection " + nClicks)); + + this.updateMessage(("Importing from " + fileType + " " + i + " of " + cpxFile.size() + " from detection " + nClicks)); //import the CPOD or FPOD data + this.updateProgress(-1, 1); importedClicks = importFile(cpxFile.get(i), tempCPODDataBlock, clickTrainDataBlock, nClicks, MAX_SAVE); - + System.out.println("Number of CPOD data units in the data block: " + tempCPODDataBlock.getUnitsCount() + " progress: " + (i+1) + " " + cpxFile.size() ); - + ListIterator iterator = tempCPODDataBlock.getListIterator(0); - + //save the click trains to the database clickTrainDataBlock.saveViewerData(); - + //save the raw clicks to the binary file.s CPODClick click; double day = -1; @@ -400,48 +374,67 @@ public class CPODImporter { while (iterator.hasNext()) { if (this.isCancelled()) return -1; click = iterator.next(); - count++; - //System.out.println("Saving click: " + click.getUID()); + System.out.println("Saving click: " + click.getUID()); //new binary file every daya; cal.setTimeInMillis(click.getTimeMilliseconds()); int dayYear = cal.get(Calendar.DAY_OF_YEAR); - - if (day!=dayYear) { - this.updateProgress(i+(count/(double) importedClicks), cpxFile.size()); + if (day!=dayYear) { + + //set the progress + double progress=(((double) click.getTimeMilliseconds()-fileStartEnd[0]))/(fileStartEnd[1]-fileStartEnd[0]); + this.updateProgress(i+(progress), cpxFile.size()); + + if (day>-1) { + + //save left over CP1 data + if (cpodTrainList!=null) { + //all duplicate clicks will have been removed. Save the data now + for (int j=0; j{ this.updateMessage(("Saving file: " + timeMillisStr)); }); - + + //get a list of the currently loaded click data if this is CP3 data if (fileType.equals(CPODFileType.CP3) || fileType.equals(CPODFileType.FP3)) { - - cpodDataBlock.clearAll(); - - //load the view from the data block for that day. - long millisDayStart = roundToDay(click.getTimeMilliseconds()); - cpodDataBlock.loadViewerData(millisDayStart, millisDayStart+24*60*60*1000L-1, null); - + + cpodDataBlock.clearAll(); + + //load the view from the data block for that day. + long millisDayStart = roundToDay(click.getTimeMilliseconds()); + cpodDataBlock.loadViewerData(millisDayStart, millisDayStart+24*60*60*1000L-1, null); + //now the tricky bit - if we are loading viewer files we want to save the data from the CP3 on top of the //CP1 file but we DO NOT want to add in duplicate clicks. if (cpodDataBlock.getUnitsCount()>0) { System.out.println("There are already data units from CP1 or FP1 files " + cpodDataBlock.getUnitsCount()); - //TODO + cpodTrainList = cpodDataBlock.getDataCopy(); + Collections.sort(cpodTrainList, new CPODClickOmparator()); + cpodDataBlock.clearAll(); //just incase } - + } //write the module head @@ -452,31 +445,70 @@ public class CPODImporter { day=dayYear; } + if (cpodTrainList!=null && cpodTrainList.size()>0) { + // index of the target + int index = Collections.binarySearch( + cpodTrainList, click, new CPODClickOmparator()); + + if (index>-1) { + //the click in the cp3 file is present in the CP1 file. Save this click as it has a super detection. DO NOT save the + //CP1 click; + cpodTrainList.remove(index); + count++; + } + + } + data = cpodDataBlock.getBinaryDataSource().getPackedData(click); this.binaryStream.storeData(data.getObjectType(), click.getBasicData(), data); } - + cpodDataBlock.clearAll(); tempCPODDataBlock.clearAll(); - - + + //update number of clicks. nClicks=nClicks+MAX_SAVE+1; } - + } } catch (Exception e) { e.printStackTrace(); } - - this.updateMessage("Finished saving data units: "); + + this.updateMessage("Finished saving detections"); System.out.println("CPOD import thread finished: " + this); return 1; } - + + private long[] getFileStartEnd(File file) { + CPODFileType fileType = CPODUtils.getFileType(file); + + long[] timelims = null; + switch (fileType){ + case CP1: + case CP3: + //TODO + break; + case FP1: + case FP3: + FPODHeader header =FPODReader.readHeader(file); + if (header==null) return null; + timelims=new long[] {CPODUtils.podTimeToMillis(header.FirstLoggedMin), CPODUtils.podTimeToMillis(header.LastLoggedMin)}; + break; + default: + break; + + } + + return timelims; + } + + + /** * Rounds millis to start of da=y * @param millis @@ -484,16 +516,16 @@ public class CPODImporter { */ long roundToDay(long millis) { Date date = new Date(millis); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - calendar.set(Calendar.HOUR_OF_DAY, 0); - calendar.set(Calendar.MINUTE, 0); - calendar.set(Calendar.SECOND, 0); - calendar.set(Calendar.MILLISECOND, 0); - return calendar.getTimeInMillis(); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + calendar.set(Calendar.HOUR_OF_DAY, 0); + calendar.set(Calendar.MINUTE, 0); + calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MILLISECOND, 0); + return calendar.getTimeInMillis(); } - - + + /** * Import a FPOD file. * @param cpFile - the FP1 or FP3 file. @@ -502,65 +534,65 @@ public class CPODImporter { * @return the total number of clicks in the file. */ protected int importFPODFile(File cpFile, CPODClickDataBlock dataBlock, CPODClickTrainDataBlock clickTrainDataBlock, int from, int maxNum) { - + ArrayList fpodData = new ArrayList(); - + try { FPODReader.importFile(cpFile, fpodData, from, maxNum); - -// fileStart + nMinutes * 60000L; - - HashMap cpodClickTrains = new HashMap(); - - int nClicks = 0; - for (int i=0; i cpodClickTrains = new HashMap(); + + int nClicks = 0; + for (int i=0; i { + @Override public int compare(CPODClick s1, CPODClick s2) + { + if (s1.getTimeMilliseconds() > s2.getTimeMilliseconds()) { + return 1; + } + else if (s1.getTimeMilliseconds() < s2.getTimeMilliseconds()) { + return -1; + } + else if (s1.getTimeMilliseconds() == s2.getTimeMilliseconds()) { + + if (s1.getStartSample()>s2.getStartSample()) return 1; + else if (s1.getStartSample() clickTrains = new HashMap(); - + //the click train of the current data unit. CPODClassification cpodClassification = null; - + int wavbufpos = 0; try { while (true) { @@ -299,7 +327,7 @@ public class FPODReader { wavData = null; } - + /** * Set the classification (if FP3 file) */ @@ -324,7 +352,7 @@ public class FPODReader { } else if (toUnsigned(bufPosN[0])==250) { - + //wav data preceedes the next click if (wavData==null) { @@ -357,49 +385,49 @@ public class FPODReader { //wav data nWavClicks++; } - + else if(toUnsigned(bufPosN[0])==249) { //click train data - this is not included - for now //click train data precedes the next click - + //the train ID is unique to the minute, short trainID = toUnsigned(bufPosN[15]); - + //1 is NBHF //2 is dolphin //3 is uncalssified //4 is sonar? short species = (short) ((bufPosN[14] >>> 2) & 3); - + //quality level for the click train short qualitylevel = (short) ((bufPosN[14]) & 3); boolean echo = false; if ((bufPosN[14] & 32) == 32) { - echo = true; + echo = true; } - - + + //generate a unique train ID within the file int trainUID = Integer.valueOf(String.format("%06d", nMinutes) + String.format("%d", trainID)); - + //find the click train from the hash map - if it is not there, create a new one. - cpodClassification = clickTrains.get(trainUID); - + cpodClassification = clickTrains.get(trainUID); + if (cpodClassification==null) { cpodClassification = new CPODClassification(); cpodClassification.isEcho = echo; cpodClassification.clicktrainID = trainUID; cpodClassification.species = CPODUtils.getSpecies(species); - + clickTrains.put(trainUID, cpodClassification); - + //System.out.println("Click train ID: " + trainUID + " minutes: " + nMinutes + " species: " + species + " quality level: " + qualitylevel); } - + } else if(toUnsigned(bufPosN[0])==254){ nMinutes ++; @@ -982,7 +1010,7 @@ public class FPODReader { public int minute; private FPODWavData wavData; - + //******Click Train Info******// @@ -1000,7 +1028,7 @@ public class FPODReader { public void setClassification(CPODClassification cpodClassification) { this.cpodClassification=cpodClassification; } - + /** * Get the species classification i.e. which click train the click belongs to. * @return the species classification object. @@ -1323,8 +1351,8 @@ public class FPODReader { if (MHzArrPtr < MhzSampledArr.length) { MhzSampledArr[MHzArrPtr] = 0; } - - + + int[] waveform = Arrays.copyOf(MhzSampledArr, MHzArrPtr); //waveform is backwards so flip it. @@ -1340,7 +1368,7 @@ public class FPODReader { // String filePath = "/Users/au671271/Library/CloudStorage/GoogleDrive-macster110@gmail.com/My Drive/PAMGuard_dev/CPOD/FPOD_NunBank/0866 NunBankB 2023 06 27 FPOD_6480 file0.FP1"; String filePath = "D:\\DropBox\\PAMGuard_dev\\CPOD\\FPOD_NunBank\\0866 NunBankB 2023 06 27 FPOD_6480 file0.FP1"; -// String filePath = "D:\\DropBox\\PAMGuard_dev\\CPOD\\FPOD_NunBank\\0866 NunBankB 2023 06 27 FPOD_6480 file0.FP3"; + // String filePath = "D:\\DropBox\\PAMGuard_dev\\CPOD\\FPOD_NunBank\\0866 NunBankB 2023 06 27 FPOD_6480 file0.FP3"; File fpfile = new File(filePath);