package org.dataone.integration.it.testImplementations;
import static org.junit.Assert.assertTrue;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import org.apache.commons.collections.IteratorUtils;
import org.dataone.client.RetryHandler;
import org.dataone.client.exception.ClientSideException;
import org.dataone.client.v1.types.D1TypeBuilder;
import org.dataone.configuration.Settings;
import org.dataone.integration.APITestUtils;
import org.dataone.integration.ContextAwareTestCaseDataone;
import org.dataone.integration.ExampleUtilities;
import org.dataone.integration.adapters.CNCallAdapter;
import org.dataone.integration.adapters.CommonCallAdapter;
import org.dataone.integration.adapters.MNCallAdapter;
import org.dataone.integration.webTest.WebTestDescription;
import org.dataone.integration.webTest.WebTestName;
import org.dataone.service.exceptions.BaseException;
import org.dataone.service.exceptions.NotFound;
import org.dataone.service.exceptions.NotImplemented;
import org.dataone.service.exceptions.ServiceFailure;
import org.dataone.service.types.v1.AccessPolicy;
import org.dataone.service.types.v1.AccessRule;
import org.dataone.service.types.v1.Identifier;
import org.dataone.service.types.v1.Node;
import org.dataone.service.types.v1.NodeReference;
import org.dataone.service.types.v1.NodeType;
import org.dataone.service.types.v1.Permission;
import org.dataone.service.types.v1.Replica;
import org.dataone.service.types.v1.ReplicationPolicy;
import org.dataone.service.types.v1.ReplicationStatus;
import org.dataone.service.types.v1.Service;
import org.dataone.service.types.v2.SystemMetadata;
import org.dataone.service.types.v2.TypeFactory;
import org.dataone.service.util.Constants;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
/**
* This class should prove whether the following series of events is functioning correctly:
*
*
* - An object is created on an MN.
* - The system metadata is changed on the MN. - we call MN.updateSystemMetadata(). This is a synchronous call.
* - Under the hood, the MN will call CN.synchronize() to push the change out. This is a synchronous call but only tells us the request for synchronize was successful
* - Under the hood, the CN REST will call CNStorage.updateSystemMetadata() to update the CN's sysmeta
* - CN REST will also call MN.systemMetadataChanged() on notify them of new metadata.
* - The replica MNs will then call getSystemMetadata() on the authoritative MN to update their copy.
*
*
* Assumptions made:
*
*
* - We have one or more working CNs in the environment
* - We have two or more working v2 MNs in the environment
* - At least one of the MNs has synchronization disabled
* (this is so we can test if CN.synchronize() works on its own correctly, as opposed to the harbest sync task).
* - The MNs/CNs are properly registered with each other.
* - We know how long it takes the CN to sync MN data and how long it takes MNs to replicate authoritative sysmeta.
*
*
* @author Andrei
*/
public class SystemMetadataFunctionalTestImplementation extends ContextAwareTestCaseDataone {
private static final String cnSubmitter = Settings.getConfiguration().getString("dataone.it.cnode.submitter.cn", "cnDevUNM1");
private List mnList;
private List cnList;
private Node mnV2NoSync;
private static final long SYNC_WAIT_MINUTES = 15;
private static final long REPLICATION_WAIT_MINUTES = 45; // on top of the sync wait
@Before
public void setup() {
cnList = new ArrayList();
mnList = new ArrayList();
Iterator cnIter = getCoordinatingNodeIterator();
if(cnIter != null)
cnList = IteratorUtils.toList(cnIter);
assertTrue("Test requires at least one CN to function!", cnList.size() > 0);
CNCallAdapter cn = new CNCallAdapter(getSession(cnSubmitter), cnList.get(0), "v2");
// check node list, add an MN to mnList if:
// it supports v2 calls
// it supports MNReplication
try {
for(Node n : cn.listNodes().getNodeList()) {
if(n.getType() == NodeType.MN)
try {
MNCallAdapter mnCallAdapter = new MNCallAdapter(getSession(cnSubmitter), n, "v2");
Node capabilities = mnCallAdapter.getCapabilities(); // also doubles as a v2 ping
List serviceList = capabilities.getServices().getServiceList();
for (Service service : serviceList)
if(service.getName().equalsIgnoreCase("MNReplication")
&& service.getAvailable()) {
mnList.add(n);
break;
}
}
catch (Exception e1) {
log.warn("MN failed V2 getCapabilities(), skipping : " + n.getIdentifier().getValue());
}
}
} catch (Exception e) {
log.warn("Unable to fetch node list from CN: " + cn.getNodeBaseServiceUrl(), e);
}
log.info("v2 CNs available: " + cnList.size());
log.info("v2 MNs available supporting replication: " + mnList.size());
for (Node n : cnList)
log.info("CN: " + n.getBaseURL());
for (Node n : mnList)
log.info("v2 MN: " + n.getBaseURL());
assertTrue("This test requires at least two v2 MNs that are tier 4 (support replication) to work.", mnList.size() >= 2);
assertTrue("This test requires at least one CN to work.", cnList.size() >= 1);
// we need to test against an MN that has synchronize disabled
// so we can be sure that the MN->CN synchronize() call is working
// instead of the harvesting sync job
// do we already have an MN with sync turned off?
Node mnNoSync = null;
for (Node n : mnList) {
MNCallAdapter mnCallAdapter = new MNCallAdapter(getSession(cnSubmitter), n, "v2");
Node capabilities = null;
try {
capabilities = mnCallAdapter.getCapabilities();
} catch (NotImplemented | ServiceFailure | ClientSideException e) {
log.warn("Unable to get MN capabilities. "
+ "Error: " + e.getClass().getSimpleName() + " : " + e.getMessage(), e);
}
if (capabilities == null) {
log.warn("MN returned NULL capabilities from getCapabilities(). MN: "
+ n.getBaseURL());
continue;
}
boolean synchronize = capabilities.isSynchronize();
if (!synchronize) {
mnNoSync = n;
break;
}
}
if (mnNoSync != null) {
mnV2NoSync = mnNoSync;
return;
}
// no MN with sync turned off?
// disable sync on an MN
for (Node n : mnList) {
MNCallAdapter mnCallAdapter = new MNCallAdapter(getSession(cnSubmitter), n, "v2");
Node capabilities = null;
try {
capabilities = mnCallAdapter.getCapabilities();
capabilities.setSynchronize(false);
org.dataone.service.types.v2.Node v2Capabilities = null;
if (capabilities instanceof org.dataone.service.types.v2.Node)
v2Capabilities = (org.dataone.service.types.v2.Node) capabilities;
else
v2Capabilities = TypeFactory.convertTypeFromType(capabilities, org.dataone.service.types.v2.Node.class);
cn.updateNodeCapabilities(null, v2Capabilities.getIdentifier(), v2Capabilities);
mnV2NoSync = capabilities;
break;
} catch (Exception e) {
log.error("Unable to disable synchronization on MN! : " + n.getBaseURL() + "\n"
+ e.getClass().getSimpleName() + " : " + e.getMessage());
}
}
assertTrue("Environment for test must have at least one v2 MN with synchronize disabled "
+ "(so we can test if CN.synchronize() works on its own correctly).", mnV2NoSync != null);
}
@After
public void tearDown() {
if(mnV2NoSync == null) {
log.error("Cannot reenable sync on MN, no reference to MN with sync disabled is saved! Reenable in LDAP.");
return;
}
Node cNode = cnList.get(0);
CNCallAdapter cn = new CNCallAdapter(getSession(cnSubmitter), cNode, "v2");
try {
NodeReference nodeRef = mnV2NoSync.getIdentifier();
org.dataone.service.types.v2.Node currentCapabilities = cn.getNodeCapabilities(nodeRef);
currentCapabilities.setSynchronize(true);
cn.updateNodeCapabilities(null, nodeRef, currentCapabilities);
} catch (Exception e) {
throw new AssertionError("Unable to update MN capabilities to re-enable synchronization on: " + mnV2NoSync.getIdentifier().getValue() +
" Reenable in LDAP.");
}
}
@Override
protected String getTestDescription() {
return "Tests the MN-triggered system metadata updating.";
}
@WebTestName("systemMetadataChanged: tests that changes made to system metadata by an MN are propegated")
@WebTestDescription("This test needs to be run in an environment with two or more MNs, one of which must "
+ "have synchronization disabled (so we can test if CN.synchronize() works on its own correctly)."
+ "The test checks whether the following events on the MN trigger the correct changes: "
+ "Some metadata is created and changed on an MN, so when the CN sees it, it'll be a new object from its point of view."
+ "The MN calls CN.synchronize() to notify the CN to update its version of the object. (Asynchronous call.)"
+ "We then need to wait for the CN to synchronize."
+ "When that happens, the CN should update its own copy, then propegate the change to "
+ "the replica MNs. We check that the CN's as well as the other MNs' copies are up to date.")
@Test
public void testSystemMetadataChanged() {
Identifier createdPid = null;
MNCallAdapter mn = new MNCallAdapter(getSession("testRightsHolder"), mnV2NoSync, "v2");
try {
// create a test object
log.info("testSystemMetadataChanged_ExistingObj: "
+ "creating test object");
AccessRule accessRule = APITestUtils.buildAccessRule(Constants.SUBJECT_PUBLIC, Permission.READ);
ReplicationPolicy replPolicy = new ReplicationPolicy();
replPolicy.setNumberReplicas(null);
final Identifier pid = D1TypeBuilder.buildIdentifier("testSystemMetadataChanged_" + ExampleUtilities.generateIdentifier());
SystemMetadata sysmeta = null;
Date originalDateModified = null;
try {
createdPid = createTestObject(mn, pid, accessRule, replPolicy);
Thread.sleep(10000);
sysmeta = mn.getSystemMetadata(null, pid);
} catch (BaseException be) {
handleFail(mn.getLatestRequestUrl(), "Unable to create a test object: " + pid.getValue());
throw be;
}
originalDateModified = sysmeta.getDateSysMetadataModified();
log.info("testSystemMetadataChanged: "
+ "created test object: " + createdPid.getValue());
// modify the sysmeta
log.info("testSystemMetadataChanged: "
+ "fetching sysmeta for pid " + createdPid.getValue() + " from auth MN");
sysmeta = mn.getSystemMetadata(null, createdPid);
log.info("testSystemMetadataChanged: "
+ "original sysmeta.serialVersion : " + sysmeta.getSerialVersion());
log.info("testSystemMetadataChanged: "
+ "original sysmeta.dateSystemMetadataChanged: " + sysmeta.getDateSysMetadataModified());
log.info("testSystemMetadataChanged: "
+ "updating sysmeta on auth MN");
boolean success = false;
try {
success = mn.updateSystemMetadata(null, createdPid, sysmeta);
} catch (BaseException be) {
handleFail(mn.getLatestRequestUrl(), "Call to MN updateSystemMetadata failed: " + be.getMessage());
throw be;
}
assertTrue("MN should have modified its own system metadata successfully.", success);
log.info("testSystemMetadataChanged: "
+ "updated sysmeta on auth MN successfully");
// MN.updateSystemMetadata() call should trigger a
// CN.synchronize() call under the hood
Node cnNode = cnList.get(0);
final CNCallAdapter cn = new CNCallAdapter(getSession(cnSubmitter), cnNode, "v2");
// CN needs time to synchronize
log.info("testSystemMetadataChanged: "
+ "polling the CN for the systemMetadata every 30 seconds while CN.synchronize() to run. " +
"(up to " + SYNC_WAIT_MINUTES + " minutes)");
final Identifier pollingPid = createdPid;
RetryHandler handler = new RetryHandler() {
@Override
protected SystemMetadata attempt()
throws TryAgainException, Exception {
try {
log.info("attempting CN getSystemMEtadata...");
return cn.getSystemMetadata(null, pollingPid);
} catch (NotFound | ServiceFailure e) {
TryAgainException f = new TryAgainException();
f.initCause(e);
throw f;
}
}
};
SystemMetadata fetchedCNSysmeta = handler.execute(30* 1000, SYNC_WAIT_MINUTES * 60 * 1000);
log.info("testSystemMetadataChanged: "
+ "done waiting for CN.synchronize(), verifying CN has correct sysmeta");
// verify that sysmeta fetched from CN is updated
boolean dateModified = fetchedCNSysmeta.getDateSysMetadataModified().after(originalDateModified);
assertTrue("testSystemMetadataChanged: sysmeta.dateSysMetadataModified should have been modified by the time it's synced to CN", dateModified );
log.info("testSystemMetadataChanged: "
+ "CN sysmeta matches updates made on MN");
// TODO could also inspect the system metadata on the CN
// for a replica in 'requested' status
// and wait for it to move to 'completed' or 'failed'
// CN needs to run replication in order for sysmeta to contain replica info
// we need replica info in the sysmeta
// so we can then verify that the sysmeta on the replica-holding MNs is updated
log.info("testSystemMetadataChanged: "
+ "waiting for CN to trigger replication. polling the CN for the systemMetadata every 30 seconds until "
+ "we have replica info in sysmeta. (waiting up to " + REPLICATION_WAIT_MINUTES + " minutes)");
// final Identifier pollingPid2 = createdPid;
RetryHandler handler2 = new RetryHandler() {
@Override
protected SystemMetadata attempt()
throws TryAgainException, Exception {
try {
log.info("attempting CN getSystemMEtadata...");
SystemMetadata sysmeta = cn.getSystemMetadata(null, pid);
log.info("attempting to get replicas from CN sysmeta...");
List replicaList = sysmeta.getReplicaList();
if (replicaList.size() == 0) {
TryAgainException f = new TryAgainException();
f.initCause(new NotFound("404", "CN sysmeta contained an empty replica list! " + cn.getLatestRequestUrl()));
throw f;
}
Node replicaNode = null;
for (Replica rep : replicaList) {
for (Node v2Node : mnList)
if (v2Node.getIdentifier().getValue().equals( rep.getReplicaMemberNode().getValue() ))
replicaNode = v2Node;
}
if (replicaNode == null) {
TryAgainException f = new TryAgainException();
f.initCause(new NotFound("404", "CN sysmeta contained no replica for a v2 MN! " + cn.getLatestRequestUrl()));
throw f;
}
return sysmeta;
} catch (NotFound | ServiceFailure e) {
TryAgainException f = new TryAgainException();
f.initCause(e);
throw f;
}
}
};
fetchedCNSysmeta = handler2.execute(30* 1000, REPLICATION_WAIT_MINUTES * 60 * 1000);
log.info("testSystemMetadataChanged: "
+ "done waiting for replication, fetching sysmeta for pid " + createdPid.getValue() + " from CN");
fetchedCNSysmeta = cn.getSystemMetadata(null, createdPid);
List replicaList = fetchedCNSysmeta.getReplicaList();
assertTrue("System metadata fetched from CN should now have a non-empty replica list", replicaList.size() > 0);
log.info("testSystemMetadataChanged: "
+ "looking at MNs that have a replica");
// notify replica-holding MNs of the sysmeta change
int replicasFound = 0;
for (Replica replica : replicaList) {
NodeReference replicaNodeRef = replica.getReplicaMemberNode();
Node replicaHolderNode = null;
for (Node n : mnList)
if (n.getIdentifier().getValue().equals(replicaNodeRef.getValue())) {
replicaHolderNode = n;
log.info("testSystemMetadataChanged: "
+ "found replica MN: " + replicaNodeRef.getValue());
break;
}
if (replicaHolderNode == null)
continue; // may be a CN NodeRef, skip
CommonCallAdapter replicaHolderMN = new CommonCallAdapter(getSession("testRightsHolder"), replicaHolderNode, "v2");
// it's not the MN's responsibility to update replica-holders
// (it's the CN's, as part of synchronize() and ensuing replication)
// so here we just check if the replicas have the updated version of sysmeta
log.info("testSystemMetadataChanged_ExistingObj: "
+ "checking if replica MN holds updated sysmeta");
SystemMetadata replicaSysmeta = replicaHolderMN.getSystemMetadata(null, createdPid);
replicasFound++;
}
assertTrue("Should have found at least one replica.", replicasFound > 0);
} catch (Exception e) {
log.error("Testing failed with exception: " + e.getClass().getSimpleName() + " : " + e.getMessage());
} finally {
// TODO ideally, purge(pid)
try {
if(createdPid != null)
mn.delete(null, createdPid);
} catch (Exception e2) {
log.warn("Unable to delete test pid after running the test: " + createdPid.getValue(), e2);
}
}
}
@WebTestName("systemMetadataChanged: tests that changes made to system metadata by an MN are propegated when the object is not a new one")
@WebTestDescription("This test needs to be run in an environment with two or more MNs, one of which must "
+ "have synchronization disabled (so we can test if CN.synchronize() works on its own correctly)."
+ "The test checks whether the following events on the MN trigger the correct changes: "
+ "Some metadata is created on an MN then waits for the CN to sync this data. The data is then modified on the MN."
+ "The MN calls CN.synchronize() to notify the CN to update its version of the object. (Asynchronous call.)"
+ "We then need to wait for the CN to synchronize."
+ "When that happens, the CN should update its own copy, then propegate the change to "
+ "the replica MNs. We check that the CN's as well as the other MNs' copies are up to date.")
@Test
public void testSystemMetadataChanged_ExistingObj() {
Identifier createdPid = null;
MNCallAdapter mn = new MNCallAdapter(getSession("testRightsHolder"), mnV2NoSync, "v2");
Node cnNode = cnList.get(0);
final CNCallAdapter cn = new CNCallAdapter(getSession(cnSubmitter), cnNode, "v2");
try {
// create a test object
log.info("testSystemMetadataChanged_ExistingObj: "
+ "creating test object");
AccessRule accessRule = APITestUtils.buildAccessRule(Constants.SUBJECT_PUBLIC, Permission.READ);
ReplicationPolicy replPolicy = new ReplicationPolicy();
replPolicy.setNumberReplicas(null);
final Identifier pid = D1TypeBuilder.buildIdentifier("testSystemMetadataChanged_ExistingObj_" + ExampleUtilities.generateIdentifier());
SystemMetadata sysmeta = null;
Date originalDateModified = null;
try {
createdPid = createTestObject(mn, pid, accessRule, replPolicy);
Thread.sleep(10000);
sysmeta = mn.getSystemMetadata(null, pid);
} catch (BaseException be) {
handleFail(mn.getLatestRequestUrl(), "Unable to create a test object: " + pid.getValue());
throw be;
}
originalDateModified = sysmeta.getDateSysMetadataModified();
log.info("testSystemMetadataChanged_ExistingObj: "
+ "created test object: " + createdPid.getValue());
// wait for CN to synchronize
// - we're testing the case where the object already exists on the CN
// at the time when CN.synchronize() gets called
log.info("testSystemMetadataChanged_ExistingObj: "
+ "waiting for CN sync (up to " + SYNC_WAIT_MINUTES + " minutes)");
RetryHandler handler = new RetryHandler() {
@Override
protected SystemMetadata attempt()
throws TryAgainException, Exception {
try {
log.info("attempting CN getSystemMetadata...");
return cn.getSystemMetadata(null, pid);
} catch (NotFound | ServiceFailure e) {
TryAgainException f = new TryAgainException();
f.initCause(e);
throw f;
}
}
};
SystemMetadata fetchedCNSysmeta = handler.execute(30* 1000, SYNC_WAIT_MINUTES * 60 * 1000);
assertTrue("cn.getSystemMetadata() should successfully fetch the sysmeta for pid " + pid.getValue() + " from " + cn.getLatestRequestUrl(), fetchedCNSysmeta != null);
log.info("testSystemMetadataChanged_ExistingObj: "
+ "fetched sysmeta for pid " + createdPid.getValue() + " from CN successfully");
// modify the sysmeta
log.info("testSystemMetadataChanged_ExistingObj: "
+ "fetching sysmeta for pid " + createdPid.getValue() + " from auth MN");
sysmeta = mn.getSystemMetadata(null, createdPid);
accessRule = APITestUtils.buildAccessRule(getSubject("testRightsHolder").getValue(), Permission.CHANGE_PERMISSION);
AccessPolicy accessPolicy = new AccessPolicy();
accessPolicy.addAllow(accessRule);
sysmeta.setAccessPolicy(accessPolicy);
log.info("testSystemMetadataChanged_ExistingObj: "
+ "new sysmeta.serialVersion : " + sysmeta.getSerialVersion());
log.info("testSystemMetadataChanged_ExistingObj: "
+ "updating sysmeta on auth MN");
boolean success = false;
try {
success = mn.updateSystemMetadata(null, createdPid, sysmeta);
} catch (BaseException be) {
handleFail(mn.getLatestRequestUrl(), "Call to MN updateSystemMetadata failed: " + be.getMessage());
throw be;
}
SystemMetadata mnSysmetaModMaybe = mn.getSystemMetadata(null, pid);
log.info("after updSysmeta, date is: " + mnSysmetaModMaybe.getDateSysMetadataModified());
assertTrue("MN should have modified its own system metadata successfully.", success);
log.info("testSystemMetadataChanged_ExistingObj: "
+ "updated sysmeta on auth MN successfully");
// MN.updateSystemMetadata() call should trigger a
// CN.synchronize() call under the hood
// CN needs time to synchronize
log.info("testSystemMetadataChanged_ExistingObj: "
+ "waiting for CN.synchronize() to run (up to " + SYNC_WAIT_MINUTES + " minutes)");
handler = new RetryHandler() {
@Override
protected SystemMetadata attempt()
throws TryAgainException, Exception {
try {
log.info("attempting CN getSystemMEtadata...");
return cn.getSystemMetadata(null, pid);
} catch (NotFound | ServiceFailure e) {
TryAgainException f = new TryAgainException();
f.initCause(e);
throw f;
}
}
};
fetchedCNSysmeta = handler.execute(30* 1000, SYNC_WAIT_MINUTES * 60 * 1000);
assertTrue("cn.getSystemMetadata() should successfully fetch the sysmeta for pid " + pid.getValue() + " from " + cn.getLatestRequestUrl(), fetchedCNSysmeta != null);
// verify that sysmeta fetched from CN is updated
boolean dateModified = fetchedCNSysmeta.getDateSysMetadataModified().after(originalDateModified);
assertTrue("testSystemMetadataChanged_ExistingObj: sysmeta.dateSysMetadataModified should have been modified by the time it's synced to CN", dateModified );
// CN needs to run replication in order for sysmeta to contain replica info
// we need replica info in the sysmeta
// so we can then verify that the sysmeta on the replica-holding MNs is updated
log.info("testSystemMetadataChanged_ExistingObj: "
+ "waiting for CN to trigger replication "
+ "so we have replica info in sysmeta (up to " + REPLICATION_WAIT_MINUTES + " minutes)");
handler = new RetryHandler() {
@Override
protected SystemMetadata attempt() throws TryAgainException, Exception {
try {
log.info("attempting CN getSystemMEtadata...");
SystemMetadata sysmeta = cn.getSystemMetadata(null, pid);
log.info("attempting to get replicas from CN sysmeta...");
List replicaList = sysmeta.getReplicaList();
if (replicaList.size() == 0) {
TryAgainException f = new TryAgainException();
f.initCause(new NotFound("404", "CN sysmeta contained an empty replica list! " + cn.getLatestRequestUrl()));
throw f;
}
Node replicaNode = null;
for (Replica rep : replicaList) {
for (Node v2Node : mnList)
if (v2Node.getIdentifier().getValue().equals( rep.getReplicaMemberNode().getValue() ))
replicaNode = v2Node;
}
if (replicaNode == null) {
TryAgainException f = new TryAgainException();
f.initCause(new NotFound("404", "CN sysmeta contained no replica for a v2 MN! " + cn.getLatestRequestUrl()));
throw f;
}
return sysmeta;
} catch (NotFound | ServiceFailure e) {
TryAgainException f = new TryAgainException();
f.initCause(e);
throw f;
}
}
};
fetchedCNSysmeta = handler.execute(30* 1000, REPLICATION_WAIT_MINUTES * 60 * 1000);
log.info("testSystemMetadataChanged_ExistingObj: "
+ "done waiting for replication, fetched sysmeta for pid " + createdPid.getValue() + " from CN");
List replicaList = fetchedCNSysmeta.getReplicaList();
assertTrue("Replica list fetched from CN (after having time to replicate should not be null!", replicaList != null);
assertTrue("System metadata fetched from CN should now have a non-empty replica list", replicaList.size() > 0);
log.info("testSystemMetadataChanged_ExistingObj: "
+ "looking at MNs that have a replica");
// check if replicas have updated sysmeta
int replicasFound = 0;
for (Replica replica : replicaList) {
NodeReference replicaNodeRef = replica.getReplicaMemberNode();
Node replicaHolderNode = null;
for (Node n : mnList)
if (n.getIdentifier().getValue().equals(replicaNodeRef.getValue())) {
replicaHolderNode = n;
log.info("testSystemMetadataChanged_ExistingObj: "
+ "found replica MN: " + replicaNodeRef.getValue());
break;
}
if (replicaHolderNode == null)
continue; // may be a CN NodeRef, skip
CommonCallAdapter replicaHolderMN = new CommonCallAdapter(getSession("testRightsHolder"), replicaHolderNode, "v2");
// it's not the MN's responsibility to update replica-holders
// (it's the CN's, as part of synchronize() and ensuing replication)
// so here we just check if the replicas have the updated version of sysmeta
log.info("testSystemMetadataChanged_ExistingObj: "
+ "checking if replica MN holds updated sysmeta");
SystemMetadata replicaSysmeta = replicaHolderMN.getSystemMetadata(null, createdPid);
replicasFound++;
}
assertTrue("Should have found at least one replica.", replicasFound > 0);
} catch (Exception e) {
log.error("Testing failed with exception: " + e.getClass().getSimpleName() + " : " + e.getMessage());
} finally {
// TODO ideally, purge(pid)
try {
if(createdPid != null)
mn.delete(null, createdPid);
} catch (Exception e2) {
log.warn("Unable to delete test pid after running the test: " + createdPid.getValue(), e2);
}
}
}
@WebTestName("setReplicationStatus: calling CN.setReplicationStatus() doesn't succeed regardless of status")
@WebTestDescription("Calling setReplicaStatus with an unchanging status should not succeed - the update should "
+ "only be valid if updating the replica status to a new one.")
@Test
public void testSetReplicationStatus_NoChange() {
// documentation says setReplicationStatus can be called by another CN,
// using CN cert since no MN certs available ... hope it works ...
Identifier createdPid = null;
MNCallAdapter mn = new MNCallAdapter(getSession("testRightsHolder"), mnV2NoSync, "v2");
Node cnNode = cnList.get(0);
final CNCallAdapter cn = new CNCallAdapter(getSession(cnSubmitter), cnNode, "v2");
try {
// create a test object
log.info("testSetReplicationStatus_NoChange: "
+ "creating test object");
AccessRule accessRule = APITestUtils.buildAccessRule(Constants.SUBJECT_PUBLIC, Permission.READ);
ReplicationPolicy replPolicy = new ReplicationPolicy();
replPolicy.setNumberReplicas(null);
final Identifier pid = D1TypeBuilder.buildIdentifier("testSetReplicationStatus_NoChange" + ExampleUtilities.generateIdentifier());
SystemMetadata sysmeta = null;
Date originalDateModified = null;
try {
createdPid = createTestObject(mn, pid, accessRule, replPolicy);
Thread.sleep(10000);
sysmeta = mn.getSystemMetadata(null, createdPid);
originalDateModified = sysmeta.getDateSysMetadataModified();
sysmeta.getAccessPolicy().addAllow(APITestUtils.buildAccessRule(Constants.SUBJECT_PUBLIC + "-2", Permission.READ));
mn.updateSystemMetadata(null, createdPid, sysmeta);
} catch (BaseException be) {
handleFail(mn.getLatestRequestUrl(), "Unable to create a test object: " + pid.getValue());
throw be;
}
log.info("testSetReplicationStatus_NoChange: "
+ "created test object: " + createdPid.getValue());
// wait for CN to synchronize
RetryHandler handler = new RetryHandler() {
@Override
protected SystemMetadata attempt()
throws TryAgainException, Exception {
try {
log.info("attempting CN getSystemMetadata...");
return cn.getSystemMetadata(null, pid);
} catch (NotFound | ServiceFailure e) {
TryAgainException f = new TryAgainException();
f.initCause(e);
throw f;
}
}
};
SystemMetadata fetchedCNSysmeta = handler.execute(30* 1000, SYNC_WAIT_MINUTES * 60 * 1000);
assertTrue("cn.getSystemMetadata() should successfully fetch the sysmeta for pid " + pid.getValue() + " from " + cn.getLatestRequestUrl(), fetchedCNSysmeta != null);
log.info("testSetReplicationStatus_NoChange: "
+ "fetched sysmeta for pid " + createdPid.getValue() + " from CN successfully");
// modify the sysmeta
log.info("testSetReplicationStatus_NoChange: "
+ "fetching sysmeta for pid " + createdPid.getValue() + " from auth MN");
sysmeta = mn.getSystemMetadata(null, createdPid);
log.info("testSetReplicationStatus_NoChange: "
+ "original sysmeta.dateSystemMetadataChanged: " + sysmeta.getDateSysMetadataModified());
log.info("testSetReplicationStatus_NoChange: "
+ "original sysmeta.serialVersion : " + sysmeta.getSerialVersion());
log.info("testSetReplicationStatus_NoChange: "
+ "updating sysmeta on auth MN");
boolean success = false;
try {
success = mn.updateSystemMetadata(null, createdPid, sysmeta);
} catch (BaseException be) {
handleFail(mn.getLatestRequestUrl(), "Call to MN updateSystemMetadata failed: " + be.getMessage());
throw be;
}
assertTrue("MN should have modified its own system metadata successfully.", success);
log.info("testSetReplicationStatus_NoChange: "
+ "updated sysmeta on auth MN successfully");
// MN.updateSystemMetadata() call should trigger a
// CN.synchronize() call under the hood
// CN needs time to synchronize
log.info("testSetReplicationStatus_NoChange: "
+ "waiting for CN.synchronize() to run (up to " + SYNC_WAIT_MINUTES + " minutes)");
handler = new RetryHandler() {
@Override
protected SystemMetadata attempt()
throws TryAgainException, Exception {
try {
log.info("attempting CN getSystemMEtadata...");
return cn.getSystemMetadata(null, pid);
} catch (NotFound | ServiceFailure e) {
TryAgainException f = new TryAgainException();
f.initCause(e);
throw f;
}
}
};
fetchedCNSysmeta = handler.execute(30* 1000, SYNC_WAIT_MINUTES * 60 * 1000);
assertTrue("cn.getSystemMetadata() should successfully fetch the sysmeta for pid " + pid.getValue() + " from " + cn.getLatestRequestUrl(), fetchedCNSysmeta != null);
// verify that sysmeta fetched from CN is updated
boolean dateModified = fetchedCNSysmeta.getDateSysMetadataModified().after(originalDateModified);
assertTrue("testSetReplicationStatus_NoChange: sysmeta.dateSysMetadataModified should have been modified by the time it's synced to CN", dateModified );
// CN needs to run replication in order for sysmeta to contain replica info
// we need replica info in the sysmeta
// so we can then verify that the sysmeta on the replica-holding MNs is updated
log.info("testSetReplicationStatus_NoChange: "
+ "waiting for CN to trigger replication "
+ "so we have replica info in sysmeta (up to " + REPLICATION_WAIT_MINUTES + " minutes)");
handler = new RetryHandler() {
@Override
protected SystemMetadata attempt() throws TryAgainException, Exception {
try {
log.info("attempting CN getSystemMEtadata...");
SystemMetadata sysmeta = cn.getSystemMetadata(null, pid);
log.info("attempting to get replicas from CN sysmeta...");
List replicaList = sysmeta.getReplicaList();
if (replicaList.size() == 0) {
TryAgainException f = new TryAgainException();
f.initCause(new NotFound("404", "CN sysmeta contained an empty replica list! " + cn.getLatestRequestUrl()));
throw f;
}
Node replicaNode = null;
for (Replica rep : replicaList) {
for (Node v2Node : mnList)
if (v2Node.getIdentifier().getValue().equals( rep.getReplicaMemberNode().getValue() ))
replicaNode = v2Node;
}
if (replicaNode == null) {
TryAgainException f = new TryAgainException();
f.initCause(new NotFound("404", "CN sysmeta contained no replica for a v2 MN! " + cn.getLatestRequestUrl()));
throw f;
}
return sysmeta;
} catch (NotFound | ServiceFailure e) {
TryAgainException f = new TryAgainException();
f.initCause(e);
throw f;
}
}
};
fetchedCNSysmeta = handler.execute(30* 1000, REPLICATION_WAIT_MINUTES * 60 * 1000);
log.info("testSetReplicationStatus_NoChange: "
+ "done waiting for replication, fetched sysmeta for pid " + createdPid.getValue() + " from CN");
List replicaList = fetchedCNSysmeta.getReplicaList();
assertTrue("Test can't continue if replica list fetched from CN (after having time to replicate) is null!", replicaList != null);
assertTrue("Test can't continue if system metadata fetched from CN does not have a non-empty replica list", replicaList.size() > 0);
// request replication a bunch
int statusUpdateFailures = 0;
for (int i = 0; i < 10; i++) {
try {
log.info("testSetReplicationStatus_NoChange: "
+ "calling setReplicationStatus (iteration " + i + ")");
cn.setReplicationStatus(null, createdPid, mn.getNodeId(), ReplicationStatus.REQUESTED, new ServiceFailure("0101", "Setting replication status to INVALIDATED failed"));
} catch (BaseException e) {
statusUpdateFailures++;
log.info("testSetReplicationStatus_NoChange: "
+ "status update failure (iteration " + i + ") "
+ e.getClass().getSimpleName() + " : " + e.getMessage());
e.printStackTrace();
}
}
log.info("testSetReplicationStatus_NoChange: "
+ "status update failures: " + statusUpdateFailures);
fetchedCNSysmeta = cn.getSystemMetadata(null, createdPid);
BigInteger serialVersion = fetchedCNSysmeta.getSerialVersion();
assertTrue("sysmeta should not have been updated if the ReplicationStatus is unchanged, "
+ "and serialVersion should not have been incremented for each attempt", serialVersion.intValue() < 10);
log.info("testSetReplicationStatus_NoChange: "
+ "status update failures: " + statusUpdateFailures);
} catch (Exception e) {
log.error("Testing failed with exception: " + e.getClass().getSimpleName() + " : " + e.getMessage());
} finally {
// TODO ideally, purge(pid)
try {
if(createdPid != null)
mn.delete(null, createdPid);
} catch (Exception e2) {
log.warn("Unable to delete test pid after running the test: " + createdPid.getValue(), e2);
}
}
}
}