From 5f9d29e1313d9d69ae39af6d246e413fc7b58db6 Mon Sep 17 00:00:00 2001 From: Douglas Gillespie <50671166+douggillespie@users.noreply.github.com> Date: Tue, 21 Feb 2023 16:01:50 +0000 Subject: [PATCH] Reprocessing Ironed out a few bugs in restarting processing. --- src/Acquisition/FileInputSystem.java | 7 ++++- src/Acquisition/FolderInputSystem.java | 3 ++ src/PamController/PamController.java | 13 +++++---- .../fileprocessing/ReprocessManager.java | 4 +++ src/RightWhaleEdgeDetector/RWEProcess.java | 1 + src/binaryFileStorage/BinaryOutputStream.java | 3 ++ src/binaryFileStorage/BinaryStore.java | 1 + src/binaryFileStorage/BinaryStoreDeleter.java | 16 +++++++---- .../BinaryStoreStatusFuncs.java | 28 ++++++++++++++----- 9 files changed, 58 insertions(+), 18 deletions(-) diff --git a/src/Acquisition/FileInputSystem.java b/src/Acquisition/FileInputSystem.java index 34f49ffa..2727ca05 100644 --- a/src/Acquisition/FileInputSystem.java +++ b/src/Acquisition/FileInputSystem.java @@ -558,7 +558,10 @@ public class FileInputSystem extends DaqSystem implements ActionListener, PamSe // if (fileInputParameters.recentFiles == null) return false; // if (fileInputParameters.recentFiles.size() < 1) return false; // String fileName = fileInputParameters.recentFiles.get(0); - return runFileAnalysis(); + if (runFileAnalysis() == false) { + return false; + } + return true; } public File getCurrentFile() { @@ -737,6 +740,8 @@ public class FileInputSystem extends DaqSystem implements ActionListener, PamSe @Override public boolean startSystem(AcquisitionControl daqControl) { + + if (audioStream == null) return false; dontStop = true; diff --git a/src/Acquisition/FolderInputSystem.java b/src/Acquisition/FolderInputSystem.java index 9ac7778c..19aff820 100644 --- a/src/Acquisition/FolderInputSystem.java +++ b/src/Acquisition/FolderInputSystem.java @@ -141,6 +141,7 @@ public class FolderInputSystem extends FileInputSystem implements PamSettings, D String[] selList = {globalFolder}; folderInputParameters.setSelectedFiles(selList); // need to immediately make the allfiles list since it's about to get used by the reprocess manager + // need to worry about how to wait for this since it's starting in a different thread. makeSelFileList(); } @@ -868,6 +869,8 @@ public class FolderInputSystem extends FileInputSystem implements PamSettings, D long fileStart = getFileStartTime(allFiles.get(i).getAbsoluteFile()); if (fileStart >= startTime) { currentFile = i; + PamCalendar.setSoundFile(true); + PamCalendar.setSessionStartTime(startTime); System.out.printf("Sound Acquisition start processing at file %s time %s\n", allFiles.get(i).getName(), PamCalendar.formatDBDateTime(fileStart)); return true; diff --git a/src/PamController/PamController.java b/src/PamController/PamController.java index 009af1bd..71f0f9ef 100644 --- a/src/PamController/PamController.java +++ b/src/PamController/PamController.java @@ -1212,6 +1212,8 @@ public class PamController implements PamControllerInterface, PamSettings { } /* + * + * This needs to be called after prepareproces. * Now we do some extra checks on the stores to see if we want to overwite data, * carry on from where we left off, etc. */ @@ -1221,14 +1223,15 @@ public class PamController implements PamControllerInterface, PamSettings { if (goonthen == false) { System.out.println("Data processing will not start since you've chosen not to overwrite existing output data"); pamStop(); + setPamStatus(PAM_IDLE); return false; } - - if (saveSettings) { - saveSettings(PamCalendar.getSessionStartTime()); - } } - + + if (saveSettings) { + saveSettings(PamCalendar.getSessionStartTime()); + } + StorageOptions.getInstance().setBlockOptions(); t1 = System.currentTimeMillis(); diff --git a/src/PamController/fileprocessing/ReprocessManager.java b/src/PamController/fileprocessing/ReprocessManager.java index 94323f67..69ebf54f 100644 --- a/src/PamController/fileprocessing/ReprocessManager.java +++ b/src/PamController/fileprocessing/ReprocessManager.java @@ -48,6 +48,10 @@ public class ReprocessManager { return true; } + if (choiceSummary.getInputStoreInfo() == null) { + return true; + } + // need to decide what to do based on the list of possible choices. ReprocessStoreChoice choice = chosePartStoreAction(choiceSummary); diff --git a/src/RightWhaleEdgeDetector/RWEProcess.java b/src/RightWhaleEdgeDetector/RWEProcess.java index 07a0584c..ecf4af68 100644 --- a/src/RightWhaleEdgeDetector/RWEProcess.java +++ b/src/RightWhaleEdgeDetector/RWEProcess.java @@ -177,6 +177,7 @@ public class RWEProcess extends PamProcess { minSoundType = rweControl.rweParameters.minSoundType; classifier.setSoundData(getSampleRate(), sourceDataBlock.getFftLength(), sourceDataBlock.getFftHop()); + System.out.println("Create right whale channel process " + iChannel); } diff --git a/src/binaryFileStorage/BinaryOutputStream.java b/src/binaryFileStorage/BinaryOutputStream.java index 6314c88e..1e474686 100644 --- a/src/binaryFileStorage/BinaryOutputStream.java +++ b/src/binaryFileStorage/BinaryOutputStream.java @@ -117,6 +117,8 @@ public class BinaryOutputStream { File outputFile = new File(mainFileName); boolean open = openPGDFFile(outputFile); + +// System.out.println("Open outout file " + outputFile.getAbsolutePath()); if (open) { addToDataMap(outputFile); @@ -260,6 +262,7 @@ public class BinaryOutputStream { public synchronized boolean closeFile() { boolean ok = true; +// System.out.println("Close output file " + mainFileName); if (dataOutputStream != null) { if (currentDataMapPoint != null) { currentDataMapPoint.setBinaryFooter(footer); diff --git a/src/binaryFileStorage/BinaryStore.java b/src/binaryFileStorage/BinaryStore.java index 3f3ebb02..f3965911 100644 --- a/src/binaryFileStorage/BinaryStore.java +++ b/src/binaryFileStorage/BinaryStore.java @@ -2404,6 +2404,7 @@ PamSettingsSource, DataOutputStore { } else { reportError("No valid header in file " + file.getAbsolutePath()); + inputStream.closeFile(); return null; } diff --git a/src/binaryFileStorage/BinaryStoreDeleter.java b/src/binaryFileStorage/BinaryStoreDeleter.java index 1497ec45..13401e04 100644 --- a/src/binaryFileStorage/BinaryStoreDeleter.java +++ b/src/binaryFileStorage/BinaryStoreDeleter.java @@ -142,9 +142,10 @@ public class BinaryStoreDeleter { boolean headOk = false; BinaryHeader binaryHead = new BinaryHeader(); try { - DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(indexFile))); + FileInputStream fis = new FileInputStream(indexFile); + DataInputStream dis = new DataInputStream(new BufferedInputStream(fis)); headOk = binaryHead.readHeader(dis); - dis.close(); + fis.close(); } catch (IOException e) { headOk = false; @@ -188,6 +189,7 @@ public class BinaryStoreDeleter { } private boolean partialCopyFile(PamDataBlock aBlock, File dataFile, long timeMillis) { + System.out.printf("Partial delete of file %s from %s\n", dataFile.getAbsoluteFile(), PamCalendar.formatDBDateTime(timeMillis)); try { BinaryInputStream inputStream = new BinaryInputStream(binaryStore, aBlock); if (inputStream.openFile(dataFile) == false) { @@ -303,10 +305,11 @@ public class BinaryStoreDeleter { * @return */ private boolean deleteFileSet(File dataFile) { + System.out.printf("Deleting full file set for %s\n", dataFile.getAbsoluteFile()); + boolean deleteOk = true; try { File indexFile = binaryStore.findIndexFile(dataFile, true); File noiseFile = binaryStore.findNoiseFile(dataFile, true); - boolean deleteOk = true; deleteOk &= dataFile.delete(); if (indexFile != null) { deleteOk &= indexFile.delete(); @@ -314,11 +317,14 @@ public class BinaryStoreDeleter { if (noiseFile != null) { deleteOk &= noiseFile.delete(); } - return deleteOk; } catch (Exception e) { - return false; + deleteOk = false; } + + System.out.printf("Deleting full file set %s for %s\n", deleteOk?"OK":"Error", dataFile.getAbsoluteFile()); + + return deleteOk; } diff --git a/src/binaryFileStorage/BinaryStoreStatusFuncs.java b/src/binaryFileStorage/BinaryStoreStatusFuncs.java index 31d9e9dc..0454655a 100644 --- a/src/binaryFileStorage/BinaryStoreStatusFuncs.java +++ b/src/binaryFileStorage/BinaryStoreStatusFuncs.java @@ -136,15 +136,16 @@ public class BinaryStoreStatusFuncs { * get the UID and time of the last item in the file. Can return these in the form of * a BinaryFooter since it's pretty much the same information needed. */ + BinaryInputStream inputStream = new BinaryInputStream(binaryStore, null); try { // need to work through the file now. - BinaryInputStream inputStream = new BinaryInputStream(binaryStore, null); if (inputStream.openFile(aFile) == false) { return null; }; BinaryObjectData binaryObjectData; BinaryHeader bh = inputStream.readHeader(); if (bh==null) { + inputStream.closeFile(); return null; } int inputFormat = bh.getHeaderFormat(); @@ -159,6 +160,7 @@ public class BinaryStoreStatusFuncs { bf = new BinaryFooter(); if (bf.readFooterData(binaryObjectData.getDataInputStream(), inputFormat)) { if (bf.getDataDate() != 0) { + inputStream.closeFile(); return bf; } } @@ -184,11 +186,18 @@ public class BinaryStoreStatusFuncs { } } } - inputStream.closeFile(); } catch (Exception e) { System.out.printf("Corrupt data file %s: %s\n", aFile, e.getMessage()); // return null; + } + try { + if (inputStream != null) { + inputStream.closeFile(); + } + } + catch (Exception e) { + } if (lastTime != null && lastUID != null) { BinaryFooter bf = new BinaryFooter(); @@ -211,18 +220,23 @@ public class BinaryStoreStatusFuncs { */ private BinaryHeader findFirstHeader(List binFiles) { BinaryHeader binaryHead = new BinaryHeader(); + DataInputStream dis = null; for (File aFile : binFiles) { try { - DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(aFile))); - if (binaryHead.readHeader(dis)) { - return binaryHead; - } + dis = new DataInputStream(new BufferedInputStream(new FileInputStream(aFile))); } catch (IOException e) { + binaryHead = null; continue; } + try { + dis.close(); + } + catch (IOException e) { + + } } - return null; + return binaryHead; } /**