Skip to content

Commit

Permalink
refactoring checksum verification to support self-healing and reporti…
Browse files Browse the repository at this point in the history 
…ng on better
  • Loading branch information
@barmintor
barmintor committed Mar 18, 2013
1 parent 4971b28 commit 316a8c5
Show file tree
Hide file tree
Showing 8 changed files with 115 additions and 122 deletions.
59 changes: 8 additions & 51 deletions fcrepo-http-api/src/main/java/org/fcrepo/api/FedoraDatastreams.java
View file
Expand Up @@ -27,6 +27,7 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
Expand Down Expand Up @@ -75,6 +76,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSet.Builder;

@Path("/objects/{pid}/datastreams")
Expand Down Expand Up @@ -482,59 +484,14 @@ public DatastreamFixity getDatastreamFixity(@PathParam("pid")
final String dsid) throws RepositoryException {

final Datastream ds = DatastreamService.getDatastream(pid, dsid);
DatastreamFixity dsf = validatedDatastreamFixity(ds);
return dsf;
}

/**
* Computes the size and sha1 of a datastream and compares
* it to that stored in the node properties
* @param ds
* @return
* @throws RepositoryException
*/
private DatastreamFixity validatedDatastreamFixity(Datastream ds) throws RepositoryException {
Node node = ds.getNode();
//compute size and checksum
//get properties for comparison
URI dsChecksum = ds.getContentDigest();
long dsSize = ds.getContentSize();

DatastreamFixity dsf = new DatastreamFixity();
MessageDigest digest = null;
try {
digest = MessageDigest.getInstance("SHA1");
} catch (NoSuchAlgorithmException e) {
logger.error("Could not locate a SHA1 provider: Everything is ruined.",e);
throw new RepositoryException(e.getMessage(),e);
}

Collection<FixityResult> blobs = getFixity(node, digest);
dsf.statuses = new ArrayList<FixityResult>(blobs.size());

for (FixityResult blob: blobs){
//FixityStatus status = new FixityStatus();
blob.validChecksum = false;
blob.validSize = false;
blob.dsChecksumType = ds.getContentDigestType();
blob.dsChecksum = dsChecksum;
blob.dsSize = dsSize;


// status.storeIdentifier = blob.storeIdentifier;
// status.computedSize = blob.computedSize;
// status.computedChecksum = blob.computedChecksum;
logger.debug("Computed checksum: {}", blob.computedChecksum);
logger.debug("Computed size is ", blob.computedSize);

if (blob.computedChecksum.equals(dsChecksum)) {
blob.validChecksum = true;
}
if (blob.computedSize == dsSize) {
blob.validSize = true;
}
dsf.statuses.add(blob);
}
dsf.objectId = pid;
dsf.dsId = dsid;
dsf.timestamp = new Date();

Collection<FixityResult> blobs = ds.runFixityAndFixProblems();
dsf.statuses = new ArrayList<FixityResult>(blobs);
return dsf;
}

Expand Down
4 changes: 2 additions & 2 deletions fcrepo-http-api/src/test/java/org/fcrepo/api/FedoraDatastreamsTest.java
View file
Expand Up @@ -324,9 +324,9 @@ public void testCheckDatastreamFixity() throws Exception {
int cache = 0;
for (FixityResult status:fixity.statuses){
logger.debug("Verifying cache {} :", cache++);
assertTrue(status.validChecksum);
assertTrue((status.status & FixityResult.BAD_CHECKSUM) != FixityResult.BAD_CHECKSUM);
logger.debug("Checksum matched");
assertTrue(status.validSize);
assertTrue((status.status & FixityResult.BAD_SIZE) != FixityResult.BAD_SIZE);
logger.debug("DS size matched");
assertTrue("Didn't find the store identifier!", compile("infinispan",
DOTALL).matcher(status.storeIdentifier).find());
Expand Down
73 changes: 39 additions & 34 deletions fcrepo-kernel/src/main/java/org/fcrepo/Datastream.java
View file
Expand Up @@ -15,7 +15,9 @@
import java.net.URI;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;

Expand Down Expand Up @@ -169,51 +171,54 @@ public String getContentDigestType() throws RepositoryException {
.getString();
}

public void runFixityAndFixProblems() throws RepositoryException {
try {
final Set<FixityResult> fixityResults = ImmutableSet.copyOf(LowLevelStorageService.getFixity(node, MessageDigest.getInstance(getContentDigestType())));
public Collection<FixityResult> runFixityAndFixProblems() throws RepositoryException {
HashSet<FixityResult> fixityResults = null;
final URI digestUri = getContentDigest();
final long size = getContentSize();
MessageDigest digest = null;
try {
digest = MessageDigest.getInstance(getContentDigestType());
} catch (NoSuchAlgorithmException e) {
throw new RepositoryException(e.getMessage(), e);
}
fixityResults = new HashSet<FixityResult>(
LowLevelStorageService.getFixity(node, digest, digestUri, size));

Set< FixityResult> goodEntries = ImmutableSet.copyOf(filter(fixityResults, new Predicate<FixityResult>() {
Set< FixityResult> goodEntries = ImmutableSet.copyOf(filter(fixityResults, new Predicate<FixityResult>() {

@Override
public boolean apply(FixityResult result) {
@Override
public boolean apply(FixityResult result) {
return result.computedChecksum.equals(digestUri) && result.computedSize == size;
}

try {
final URI digest = getContentDigest();
final long size = getContentSize();
return result.computedChecksum.equals(digest) && result.computedSize == size;
} catch (RepositoryException e) {
return false;
}
}
;
}));

;
}));
if (goodEntries.size() == 0) {
logger.error("ALL COPIES OF " + getObject().getName() + "/" + getDsId() + " HAVE FAILED FIXITY CHECKS.");
return fixityResults;
}

assert goodEntries.size() > 0;
Iterator<FixityResult> it = goodEntries.iterator();

Iterator<FixityResult> it = goodEntries.iterator();

LowLevelCacheEntry anyGoodCacheEntry = it.next().getEntry();

LowLevelCacheEntry anyGoodCacheEntry = it.next().getEntry();

Set<FixityResult> badEntries = Sets.difference(fixityResults, goodEntries);

Set<FixityResult> badEntries = Sets.difference(fixityResults, goodEntries);

for(FixityResult result : badEntries) {

try {
result.getEntry().storeValue(anyGoodCacheEntry.getInputStream());
} catch (IOException e) {
e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
}
}


} catch (NoSuchAlgorithmException e) {

}
for(FixityResult result : badEntries) {

try {
result.getEntry().storeValue(anyGoodCacheEntry.getInputStream());
FixityResult newResult = result.getEntry().checkFixity(digestUri, size, digest);
if (newResult.status == FixityResult.SUCCESS) result.status |= FixityResult.REPAIRED;
} catch (IOException e) {
e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
}
}

return fixityResults;
}

/**
Expand Down
31 changes: 6 additions & 25 deletions fcrepo-kernel/src/main/java/org/fcrepo/services/LowLevelStorageService.java
View file
Expand Up @@ -9,6 +9,7 @@

import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Collection;
Expand Down Expand Up @@ -64,7 +65,8 @@ private static JcrRepository getRepositoryInstance() {
}

public static Collection<FixityResult> getFixity(
final Node resource, final MessageDigest digest) throws RepositoryException {
final Node resource, final MessageDigest digest,
final URI dsChecksum, final long dsSize) throws RepositoryException {
logger.debug("Checking resource: " + resource.getPath());

return transformBinaryBlobs(
Expand All @@ -75,33 +77,12 @@ public FixityResult transformEntry(LowLevelCacheEntry entry,
InputStream is) {
logger.debug("Checking fixity for resource in cache store " + entry.toString());
FixityResult result = null;
FixityInputStream ds = null;
try {
ds =
new FixityInputStream(is,
(MessageDigest) digest.clone());

result = new FixityResult(entry);
result.storeIdentifier = entry.getExternalIdentifier();
//result.
while (ds.read() != -1) ;

String calculatedDigest =
encodeHexString(ds.getMessageDigest()
.digest());
result.computedChecksum = ContentDigest.asURI(digest.getAlgorithm(), calculatedDigest);
result.computedSize = ds.getByteCount();

logger.debug("Got " + result.toString());
ds.close();

} catch (CloneNotSupportedException e) {
e.printStackTrace();
} catch (IOException e) {
result = entry.checkFixity(dsChecksum, dsSize, digest);
} catch (BinaryStoreException e) {
e.printStackTrace();
throw new IllegalStateException(e);
}

}
return result;
}

Expand Down
21 changes: 14 additions & 7 deletions fcrepo-kernel/src/main/java/org/fcrepo/utils/FixityResult.java
View file
Expand Up @@ -2,14 +2,26 @@

import java.net.URI;

import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;

@XmlRootElement(name = "DatastreamFixityStatus")
public class FixityResult {

public static final int SUCCESS = 0;

public static final int REPAIRED = 1;

public static final int BAD_CHECKSUM = 2;

public static final int BAD_SIZE = 4;

@XmlElement
public String storeIdentifier;

@XmlAttribute
public int status;

@XmlElement
public long computedSize;
Expand All @@ -26,12 +38,6 @@ public class FixityResult {
@XmlElement
public URI dsChecksum;

@XmlElement
public boolean validChecksum;

@XmlElement
public boolean validSize;

private final LowLevelCacheEntry entry;

public FixityResult() {
Expand All @@ -40,6 +46,7 @@ public FixityResult() {

public FixityResult(LowLevelCacheEntry entry) {
this.entry = entry;
this.storeIdentifier = entry.getExternalIdentifier();
}

public FixityResult(long size, URI checksum) {
Expand All @@ -53,7 +60,7 @@ public FixityResult(LowLevelCacheEntry entry, long size, URI checksum) {
this.computedSize = size;
this.computedChecksum = checksum;
}

public boolean equals(Object obj) {

boolean result = false;
Expand Down
42 changes: 42 additions & 0 deletions fcrepo-kernel/src/main/java/org/fcrepo/utils/LowLevelCacheEntry.java
View file
@@ -1,8 +1,12 @@
package org.fcrepo.utils;

import static org.apache.commons.codec.binary.Hex.encodeHexString;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.security.MessageDigest;
import java.util.Properties;

import org.apache.poi.util.IOUtils;
Expand All @@ -16,9 +20,14 @@
import org.modeshape.jcr.value.binary.BinaryStore;
import org.modeshape.jcr.value.binary.BinaryStoreException;
import org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class LowLevelCacheEntry {

private static final Logger logger = LoggerFactory
.getLogger(LowLevelCacheEntry.class);

private static final String DATA_SUFFIX = "-data";
private final BinaryStore store;
private final CacheStore low_level_store;
Expand Down Expand Up @@ -97,4 +106,37 @@ public String getExternalIdentifier() {
return this.store.toString();
}
}

public FixityResult checkFixity(URI checksum, long size, MessageDigest digest) throws BinaryStoreException {
FixityResult result = null;
FixityInputStream ds = null;
try {
ds =
new FixityInputStream(getInputStream(),
(MessageDigest) digest.clone());

result = new FixityResult(this);
//result.
while (ds.read() != -1) ;

String calculatedDigest =
encodeHexString(ds.getMessageDigest()
.digest());
result.computedChecksum = ContentDigest.asURI(digest.getAlgorithm(), calculatedDigest);
result.computedSize = ds.getByteCount();
result.dsChecksum = checksum;
result.dsSize = size;
if (!result.computedChecksum.equals(result.dsChecksum)) result.status |= FixityResult.BAD_CHECKSUM;
if (result.dsSize != result.computedSize) result.status |= FixityResult.BAD_SIZE;
logger.debug("Got " + result.toString());
ds.close();
} catch (CloneNotSupportedException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
throw new IllegalStateException(e);
}

return result;
}
}
4 changes: 3 additions & 1 deletion fcrepo-kernel/src/test/java/org/fcrepo/services/LowLevelStorageServiceTest.java
View file
Expand Up @@ -48,7 +48,9 @@ public void testChecksumBlobs() throws Exception {

final Datastream ds = getDatastream("testObject", "testRepositoryContent");

final Collection<FixityResult> fixityResults = LowLevelStorageService.getFixity(ds.getNode(), MessageDigest.getInstance("SHA-1"));
final Collection<FixityResult> fixityResults = LowLevelStorageService.getFixity(
ds.getNode(), MessageDigest.getInstance("SHA-1"),
ds.getContentDigest(), ds.getContentSize());

assertNotEquals(0, fixityResults.size());

Expand Down
3 changes: 1 addition & 2 deletions fcrepo-kernel/src/test/java/org/fcrepo/utils/SelfHealingTest.java
View file
Expand Up @@ -91,8 +91,7 @@ private void tamperWithNode(final Node node) throws Exception {

private Collection<FixityResult> getNodeFixity(final Datastream ds) throws NoSuchAlgorithmException, RepositoryException {

return getFixity(ds.getNode(), MessageDigest.getInstance("SHA-1"));

return getFixity(ds.getNode(), MessageDigest.getInstance("SHA-1"), ds.getContentDigest(), ds.getContentSize());

}

Expand Down

0 comments on commit 316a8c5

Please sign in to comment.