Skip to content

Commit

Permalink
replace fixity JAX-B response with RDF; move fixity work up to Datast…
Browse files Browse the repository at this point in the history 
…reamService, and add a RDF serialization for fixity results.
  • Loading branch information
@cbeer
cbeer committed May 21, 2013
1 parent 9b2d029 commit 99f67f7
Show file tree
Hide file tree
Showing 17 changed files with 514 additions and 360 deletions.
53 changes: 23 additions & 30 deletions fcrepo-http-api/src/main/java/org/fcrepo/api/FedoraFixity.java
View file
@@ -1,11 +1,13 @@
package org.fcrepo.api;

import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
import static javax.ws.rs.core.MediaType.TEXT_XML;
import static org.fcrepo.http.RDFMediaType.N3;
import static org.fcrepo.http.RDFMediaType.N3_ALT1;
import static org.fcrepo.http.RDFMediaType.N3_ALT2;
import static org.fcrepo.http.RDFMediaType.NTRIPLES;
import static org.fcrepo.http.RDFMediaType.RDF_JSON;
import static org.fcrepo.http.RDFMediaType.RDF_XML;
import static org.fcrepo.http.RDFMediaType.TURTLE;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.List;

import javax.jcr.RepositoryException;
Expand All @@ -14,14 +16,18 @@
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.PathSegment;
import javax.ws.rs.core.Request;
import javax.ws.rs.core.UriInfo;
import javax.ws.rs.core.Variant;

import com.hp.hpl.jena.query.Dataset;
import org.fcrepo.AbstractResource;
import org.fcrepo.Datastream;
import org.fcrepo.jaxb.responses.management.DatastreamFixity;
import org.fcrepo.api.rdf.HttpGraphSubjects;
import org.fcrepo.http.RDFMediaType;
import org.fcrepo.services.DatastreamService;
import org.fcrepo.services.LowLevelStorageService;
import org.fcrepo.utils.FixityResult;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

Expand All @@ -34,30 +40,25 @@ public class FedoraFixity extends AbstractResource {
@Autowired
private DatastreamService datastreamService;

@Autowired
private LowLevelStorageService llStoreService;

@GET
@Timed
@Produces({TEXT_XML, APPLICATION_JSON})
public DatastreamFixity getDatastreamFixity(@PathParam("path")
List<PathSegment> pathList) throws RepositoryException {
@Produces({N3, N3_ALT1, N3_ALT2, TURTLE, RDF_XML, RDF_JSON, NTRIPLES})
public Dataset getDatastreamFixity(@PathParam("path") List<PathSegment> pathList,
@Context final Request request,
@Context final UriInfo uriInfo) throws RepositoryException {

final Session session = getAuthenticatedSession();

try {
final String path = toPath(pathList);

final Datastream ds = datastreamService.getDatastream(session, path);
/* select the best response type */
final Variant bestPossibleResponse =
request.selectVariant(RDFMediaType.POSSIBLE_RDF_VARIANTS);

final DatastreamFixity dsf = new DatastreamFixity();
dsf.path = path;
dsf.timestamp = new Date();
final Datastream ds = datastreamService.getDatastream(session, path);

final Collection<FixityResult> blobs =
llStoreService.runFixityAndFixProblems(ds);
dsf.statuses = new ArrayList<FixityResult>(blobs);
return dsf;
return datastreamService.getFixityResultsModel(new HttpGraphSubjects(FedoraNodes.class, uriInfo), ds);
} finally {
session.logout();
}
Expand All @@ -71,12 +72,4 @@ public void setDatastreamService(final DatastreamService datastreamService) {
this.datastreamService = datastreamService;
}

public LowLevelStorageService getLlStoreService() {
return llStoreService;
}

public void setLlStoreService(final LowLevelStorageService llStoreService) {
this.llStoreService = llStoreService;
}

}
31 changes: 16 additions & 15 deletions fcrepo-http-api/src/test/java/org/fcrepo/api/FedoraFixityTest.java
View file
Expand Up @@ -2,22 +2,29 @@

import static org.fcrepo.test.util.PathSegmentImpl.createPathList;
import static org.junit.Assert.assertNotNull;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import java.io.IOException;
import java.security.Principal;
import java.util.Collection;

import javax.jcr.LoginException;
import javax.jcr.Node;
import javax.jcr.RepositoryException;
import javax.jcr.Session;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.core.Request;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;

import com.hp.hpl.jena.query.Dataset;
import org.fcrepo.Datastream;
import org.fcrepo.jaxb.responses.management.DatastreamFixity;
import org.fcrepo.rdf.GraphSubjects;
import org.fcrepo.services.DatastreamService;
import org.fcrepo.services.LowLevelStorageService;
import org.fcrepo.test.util.TestHelpers;
Expand All @@ -33,28 +40,23 @@ public class FedoraFixityTest {

LowLevelStorageService mockLow;

Repository mockRepo;

Session mockSession;

SecurityContext mockSecurityContext;

HttpServletRequest mockServletRequest;

Principal mockPrincipal;
private UriInfo uriInfo;
private Request mockRequest;

String mockUser = "testuser";

@Before
@Before
public void setUp() throws LoginException, RepositoryException {

mockRequest = mock(Request.class);
mockDatastreams = mock(DatastreamService.class);
mockLow = mock(LowLevelStorageService.class);

testObj = new FedoraFixity();
testObj.setDatastreamService(mockDatastreams);
testObj.setLlStoreService(mockLow);

testObj.setUriInfo(TestHelpers.getUriInfoImpl());
uriInfo = TestHelpers.getUriInfoImpl();
testObj.setUriInfo(uriInfo);


mockSession = TestHelpers.mockSession(testObj);
Expand All @@ -75,8 +77,7 @@ public void testGetDatastreamFixity() throws RepositoryException,
when(mockNode.getSession()).thenReturn(mockSession);
when(mockDs.getNode()).thenReturn(mockNode);
when(mockDatastreams.getDatastream(mockSession, path)).thenReturn(mockDs);
final DatastreamFixity actual = testObj.getDatastreamFixity(createPathList("objects", pid, "testDS"));
assertNotNull(actual);
verify(mockLow).runFixityAndFixProblems(mockDs);
testObj.getDatastreamFixity(createPathList("objects", pid, "testDS"), mockRequest, uriInfo);
verify(mockDatastreams).getFixityResultsModel(any(GraphSubjects.class), eq(mockDs));
}
}
34 changes: 15 additions & 19 deletions fcrepo-http-api/src/test/java/org/fcrepo/integration/api/FedoraFixityIT.java
View file
Expand Up @@ -11,12 +11,16 @@
import javax.xml.bind.JAXBContext;
import javax.xml.bind.Unmarshaller;

import com.hp.hpl.jena.graph.Node;
import com.hp.hpl.jena.rdf.model.ResourceFactory;
import com.hp.hpl.jena.update.GraphStore;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.util.EntityUtils;
import org.fcrepo.jaxb.responses.management.DatastreamFixity;
import org.fcrepo.test.util.TestHelpers;
import org.fcrepo.utils.FixityResult;
import org.junit.Test;

Expand All @@ -34,24 +38,16 @@ public void testCheckDatastreamFixity() throws Exception {
final HttpResponse response = execute(method2);
assertEquals(200, response.getStatusLine().getStatusCode());
final HttpEntity entity = response.getEntity();
final String content = EntityUtils.toString(entity);
final JAXBContext context =
JAXBContext.newInstance(DatastreamFixity.class);
final Unmarshaller um = context.createUnmarshaller();
final DatastreamFixity fixity =
(DatastreamFixity) um.unmarshal(new java.io.StringReader(
content));
int cache = 0;
for (final FixityResult status : fixity.statuses) {
logger.debug("Verifying cache {} :", cache++);
assertFalse(status.status.contains(BAD_CHECKSUM));
logger.debug("Checksum matched");
assertFalse(status.status.contains(BAD_SIZE));
logger.debug("DS size matched");
assertTrue("Didn't find the store identifier!", compile(
"infinispan", DOTALL).matcher(status.storeIdentifier)
.find());
logger.debug("cache store matched");
}
final GraphStore graphStore = TestHelpers.parseTriples(entity.getContent());

logger.info("Got triples {}", graphStore);

assertTrue(graphStore.contains(Node.ANY, Node.ANY, ResourceFactory.createProperty("info:fedora/fedora-system:def/internal#isFixityResultOf").asNode(), ResourceFactory.createResource(serverAddress + "objects/FedoraDatastreamsTest11/zxc").asNode()));
assertTrue(graphStore.contains(Node.ANY, Node.ANY, ResourceFactory.createProperty("info:fedora/fedora-system:def/internal#status").asNode(), ResourceFactory.createPlainLiteral("SUCCESS").asNode()));

assertTrue(graphStore.contains(Node.ANY, Node.ANY, ResourceFactory.createProperty("info:fedora/fedora-system:def/internal#computedChecksum").asNode(),ResourceFactory.createResource("urn:sha1:0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33").asNode()));
assertTrue(graphStore.contains(Node.ANY, Node.ANY, ResourceFactory.createProperty("info:fedora/fedora-system:def/internal#storedChecksum").asNode(), ResourceFactory.createResource("urn:sha1:0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33").asNode()));
assertTrue(graphStore.contains(Node.ANY, Node.ANY, ResourceFactory.createProperty("info:fedora/fedora-system:def/internal#computedSize").asNode(),ResourceFactory.createTypedLiteral(3).asNode()));
assertTrue(graphStore.contains(Node.ANY, Node.ANY, ResourceFactory.createProperty("info:fedora/fedora-system:def/internal#storedSize").asNode(), ResourceFactory.createTypedLiteral(3).asNode()));
}
}
130 changes: 130 additions & 0 deletions fcrepo-kernel/src/main/java/org/fcrepo/services/DatastreamService.java
View file
@@ -1,20 +1,40 @@

package org.fcrepo.services;

import static com.codahale.metrics.MetricRegistry.name;
import static com.google.common.collect.ImmutableSet.copyOf;
import static com.google.common.collect.Sets.difference;
import static java.security.MessageDigest.getInstance;
import static org.slf4j.LoggerFactory.getLogger;

import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Collection;
import java.util.Set;

import javax.jcr.Node;
import javax.jcr.RepositoryException;
import javax.jcr.Session;

import com.codahale.metrics.Counter;
import com.codahale.metrics.Timer;
import com.hp.hpl.jena.query.Dataset;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.update.GraphStoreFactory;
import org.fcrepo.Datastream;
import org.fcrepo.FedoraObject;
import org.fcrepo.binary.PolicyDecisionPoint;
import org.fcrepo.exception.InvalidChecksumException;
import org.fcrepo.rdf.GraphSubjects;
import org.fcrepo.services.functions.GetGoodFixityResults;
import org.fcrepo.utils.DatastreamIterator;
import org.fcrepo.utils.FixityResult;
import org.fcrepo.utils.JcrRdfTools;
import org.fcrepo.utils.LowLevelCacheEntry;
import org.modeshape.jcr.api.JcrConstants;
import org.slf4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;

Expand All @@ -29,9 +49,29 @@ public class DatastreamService extends RepositoryService {
@Autowired(required=false)
PolicyDecisionPoint storagePolicyDecisionPoint;

@Autowired
private LowLevelStorageService llStoreService;

static final Counter fixityCheckCounter = metrics.counter(name(
LowLevelStorageService.class, "fixity-check-counter"));

static final Timer timer = metrics.timer(name(Datastream.class,
"fixity-check-time"));

static final Counter fixityRepairedCounter = metrics.counter(name(
LowLevelStorageService.class, "fixity-repaired-counter"));

static final Counter fixityErrorCounter = metrics.counter(name(
LowLevelStorageService.class, "fixity-error-counter"));


private static final Logger logger = getLogger(DatastreamService.class);


private GetGoodFixityResults getGoodFixityResults =
new GetGoodFixityResults();


/**
* Create a new Datastream node in the JCR store
* @param session the jcr session to use
Expand Down Expand Up @@ -153,4 +193,94 @@ private PolicyDecisionPoint getStoragePolicyDecisionPoint() {

return storagePolicyDecisionPoint;
}

public Dataset getFixityResultsModel(final GraphSubjects factory, final Datastream datastream) throws RepositoryException {


final Collection<FixityResult> blobs = runFixityAndFixProblems(datastream);


final Model model = JcrRdfTools.getFixityResultsModel(factory, datastream.getNode(), blobs);

return GraphStoreFactory.create(model).toDataset();
}

public void setLlStoreService(final LowLevelStorageService llStoreService) {
this.llStoreService = llStoreService;
}


public Collection<FixityResult> runFixityAndFixProblems(
final Datastream datastream) throws RepositoryException {
Set<FixityResult> fixityResults;
Set<FixityResult> goodEntries;
final URI digestUri = datastream.getContentDigest();
final long size = datastream.getContentSize();
MessageDigest digest;

fixityCheckCounter.inc();

try {
digest = getInstance(datastream.getContentDigestType());
} catch (final NoSuchAlgorithmException e) {
throw new RepositoryException(e.getMessage(), e);
}

final Timer.Context context = timer.time();

try {
fixityResults =
copyOf(getFixity(datastream.getNode().getNode(JcrConstants.JCR_CONTENT), digest, digestUri,
size));

goodEntries = getGoodFixityResults.apply(fixityResults);
} finally {
context.stop();
}

if (goodEntries.size() == 0) {
logger.error("ALL COPIES OF " + datastream.getNode().getPath() + " HAVE FAILED FIXITY CHECKS.");
return fixityResults;
}

final LowLevelCacheEntry anyGoodCacheEntry =
goodEntries.iterator().next().getEntry();

final Set<FixityResult> badEntries =
difference(fixityResults, goodEntries);

for (final FixityResult result : badEntries) {
try {
result.getEntry()
.storeValue(anyGoodCacheEntry.getInputStream());
final FixityResult newResult =
result.getEntry().checkFixity(digestUri, size, digest);
if (newResult.isSuccess()) {
result.setRepaired();
fixityRepairedCounter.inc();
} else {
fixityErrorCounter.inc();
}
} catch (final IOException e) {
logger.warn("Exception repairing low-level cache entry: {}", e);
}
}

return fixityResults;
}


public Collection<FixityResult> getFixity(final Node resource, final MessageDigest digest,
final URI dsChecksum, final long dsSize)
throws RepositoryException {
logger.debug("Checking resource: " + resource.getPath());

return llStoreService.transformLowLevelCacheEntries(resource, ServiceHelpers
.getCheckCacheFixityFunction(digest, dsChecksum, dsSize));
}


public void setGetGoodFixityResults(final GetGoodFixityResults res) {
this.getGoodFixityResults = res;
}
}

0 comments on commit 99f67f7

Please sign in to comment.