Skip to content

Commit 18db47d

Browse files
lilstiffylilstiffy
committed
Update literate-octo-guacamole to 1.2 that now handles larger file sizes
1 parent 8377bd7 commit 18db47d

4 files changed

Lines changed: 70 additions & 59 deletions

File tree

‎File-Integrity-Scanner/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@
9191
<dependency>
9292
<groupId>lib.pwss</groupId>
9393
<artifactId>algorithm-hash-extraction</artifactId>
94-
<version>1.1</version>
94+
<version>1.2</version>
9595
</dependency>
9696
<!--https://github.com/pwssOrg/PWSS-DirectoryNav/packages/2588313-->
9797
<dependency>

‎File-Integrity-Scanner/src/main/java/org/pwss/file_integrity_scanner/component/FileHashComputer.java‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.pwss.file_integrity_scanner.component;
22

3-
import lib.pwss.hash.FileHashHandler;
3+
import lib.pwss.hash.file_hash_handler.BigFileHashHandler;
4+
import lib.pwss.hash.file_hash_handler.FileHashHandler;
45
import lib.pwss.hash.compare.util.HashCompareUtil;
56
import lib.pwss.hash.model.HashForFilesOutput;
67
import org.pwss.file_integrity_scanner.msr.domain.model.entities.checksum.Checksum;
@@ -14,20 +15,34 @@
1415
@Component
1516
public final class FileHashComputer {
1617

18+
private final org.slf4j.Logger log;
19+
20+
// TODO: Let the user adjust the maximum limit
21+
private final long TEMP_USER_DEFINED_MAX_LIMIT = 5000L * 1024 * 1024; // 100 MB
1722

1823
public FileHashComputer() {
24+
this.log = org.slf4j.LoggerFactory.getLogger(FileHashComputer.class);
1925
}
2026

27+
// Instance of FileHashHandler for computing hashes of smaller files
2128
private final FileHashHandler fileHashHandler = new FileHashHandler();
2229

30+
// Instance of BigFileHashHandler for computing hashes of larger files
31+
private final BigFileHashHandler bigFileHashHandler = new BigFileHashHandler(TEMP_USER_DEFINED_MAX_LIMIT);
32+
2333
/**
2434
* Computes all hashes for the given file.
2535
*
2636
* @param file the file for which hashes need to be computed
2737
* @return an object containing the computed hashes for the file
2838
*/
2939
public HashForFilesOutput computeHashes(File file) {
30-
return fileHashHandler.GetAllHashes(file);
40+
try {
41+
return fileHashHandler.GetAllHashes(file);
42+
} catch (OutOfMemoryError outOfMemoryError) {
43+
log.debug("Large file detected, switching to BigFileHashHandler for file: {}", file.getPath());
44+
return bigFileHashHandler.GetAllHashes(file);
45+
}
3146
}
3247

3348
/**

‎File-Integrity-Scanner/src/main/java/org/pwss/file_integrity_scanner/msr/service/scan/ScanServiceImpl.java‎

Lines changed: 51 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -280,66 +280,62 @@ private void processFile(File file, Scan scanInstance) {
280280
org.pwss.file_integrity_scanner.msr.domain.model.entities.file.File fileEntity;
281281
boolean fileInDatabase = fileService.existsByPath(file.getPath());
282282

283-
try {
284-
HashForFilesOutput computedHashes = fileHashComputer.computeHashes(file);
285-
286-
MonitoredDirectory mDirectory = scanInstance.getMonitoredDirectory();
287-
288-
if (fileInDatabase) {
289-
// Fetch existing entity and update fields
290-
fileEntity = fileService.findByPath(file.getPath());
291-
fileEntity.setSize(file.length());
292-
OffsetDateTime lastModified = Instant.ofEpochMilli(file.lastModified())
293-
.atOffset(ZoneOffset.UTC);
294-
fileEntity.setMtime(lastModified);
295-
log.debug("Updating existing file in DB: {}", fileEntity.getPath());
296-
} else {
297-
// Create new entity
298-
fileEntity = new org.pwss.file_integrity_scanner.msr.domain.model.entities.file.File();
299-
fileEntity.setPath(file.getPath());
300-
fileEntity.setBasename(file.getName());
301-
fileEntity.setDirectory(file.getParent());
302-
fileEntity.setSize(file.length());
303-
OffsetDateTime lastModified = Instant.ofEpochMilli(file.lastModified())
304-
.atOffset(ZoneOffset.UTC);
305-
fileEntity.setMtime(lastModified);
306-
log.debug("Adding new file to DB: {}", fileEntity.getPath());
307-
}
283+
HashForFilesOutput computedHashes = fileHashComputer.computeHashes(file);
284+
285+
MonitoredDirectory mDirectory = scanInstance.getMonitoredDirectory();
286+
287+
if (fileInDatabase) {
288+
// Fetch existing entity and update fields
289+
fileEntity = fileService.findByPath(file.getPath());
290+
fileEntity.setSize(file.length());
291+
OffsetDateTime lastModified = Instant.ofEpochMilli(file.lastModified())
292+
.atOffset(ZoneOffset.UTC);
293+
fileEntity.setMtime(lastModified);
294+
log.debug("Updating existing file in DB: {}", fileEntity.getPath());
295+
} else {
296+
// Create new entity
297+
fileEntity = new org.pwss.file_integrity_scanner.msr.domain.model.entities.file.File();
298+
fileEntity.setPath(file.getPath());
299+
fileEntity.setBasename(file.getName());
300+
fileEntity.setDirectory(file.getParent());
301+
fileEntity.setSize(file.length());
302+
OffsetDateTime lastModified = Instant.ofEpochMilli(file.lastModified())
303+
.atOffset(ZoneOffset.UTC);
304+
fileEntity.setMtime(lastModified);
305+
log.debug("Adding new file to DB: {}", fileEntity.getPath());
306+
}
308307

309-
fileService.save(fileEntity);
310-
311-
Checksum checksum = new Checksum();
312-
checksum.setChecksumSha256(computedHashes.sha256());
313-
checksum.setChecksumSha3(computedHashes.sha3());
314-
checksum.setChecksumBlake2b(computedHashes.blake2());
315-
checksum.setFile(fileEntity);
316-
checksumService.save(checksum);
317-
318-
// If the baseline is established, check if the file has changed
319-
if (monitoredDirectoryService.isBaseLineEstablished(mDirectory)) {
320-
List<Checksum> dbChecksums = checksumService.findByFile(fileEntity);
321-
if (!dbChecksums.isEmpty()) {
322-
// TODO: Maybe not getFirst but let's discuss this later
323-
Checksum oldChecksum = dbChecksums.getFirst();
324-
if (fileHashComputer.compareHashes(oldChecksum, checksum)) {
325-
log.info("File {} has not changed since last scan ✅", fileEntity.getPath());
326-
} else {
327-
log.warn("File {} has changed since last scan ⚠️", fileEntity.getPath());
328-
// TODO: Figure out what to do with changed files, add some notes to scan summary?
329-
}
308+
fileService.save(fileEntity);
309+
310+
Checksum checksum = new Checksum();
311+
checksum.setChecksumSha256(computedHashes.sha256());
312+
checksum.setChecksumSha3(computedHashes.sha3());
313+
checksum.setChecksumBlake2b(computedHashes.blake2());
314+
checksum.setFile(fileEntity);
315+
checksumService.save(checksum);
316+
317+
// If the baseline is established, check if the file has changed
318+
if (monitoredDirectoryService.isBaseLineEstablished(mDirectory)) {
319+
List<Checksum> dbChecksums = checksumService.findByFile(fileEntity);
320+
if (!dbChecksums.isEmpty()) {
321+
// TODO: Maybe not getFirst but let's discuss this later
322+
Checksum oldChecksum = dbChecksums.getFirst();
323+
if (fileHashComputer.compareHashes(oldChecksum, checksum)) {
324+
log.info("File {} has not changed since last scan ✅", fileEntity.getPath());
330325
} else {
331-
log.info("No existing checksum found for file from prior scans {}", fileEntity.getPath());
326+
log.warn("File {} has changed since last scan ⚠️", fileEntity.getPath());
327+
// TODO: Figure out what to do with changed files, add some notes to scan summary?
332328
}
329+
} else {
330+
log.info("No existing checksum found for file from prior scans {}", fileEntity.getPath());
333331
}
334-
335-
ScanSummary scanSummary = new ScanSummary();
336-
scanSummary.setFile(fileEntity);
337-
scanSummary.setScan(scanInstance);
338-
scanSummary.setChecksum(checksum);
339-
scanSummaryService.save(scanSummary);
340-
} catch (OutOfMemoryError memoryError) {
341-
log.warn("Out of memory error while processing file: {}. Skipping file.", file.getPath());
342332
}
333+
334+
ScanSummary scanSummary = new ScanSummary();
335+
scanSummary.setFile(fileEntity);
336+
scanSummary.setScan(scanInstance);
337+
scanSummary.setChecksum(checksum);
338+
scanSummaryService.save(scanSummary);
343339
}
344340

345341
@Override

‎File-Integrity-Scanner/src/test/java/org/pwss/file_integrity_scanner/PWSSLibraryTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package org.pwss.file_integrity_scanner;
22

3-
import lib.pwss.hash.FileHashHandler;
43
import lib.pwss.hash.compare.util.HashCompareUtil;
4+
import lib.pwss.hash.file_hash_handler.FileHashHandler;
55
import lib.pwss.hash.model.HashForFilesOutput;
66
import org.junit.jupiter.api.Assertions;
77
import org.junit.jupiter.api.BeforeEach;

0 commit comments

Comments
 (0)