package edu.ucsb.nceas.metacat.admin.upgrade.dataone; /** * '$RCSfile$' * Purpose: A Class for upgrading the database to version 1.5 * Copyright: 2000 Regents of the University of California and the * National Center for Ecological Analysis and Synthesis * Authors: Saurabh Garg * * '$Author: leinfelder $' * '$Date: 2011-03-29 18:23:38 +0000 (Tue, 29 Mar 2011) $' * '$Revision: 6025 $' * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import edu.ucsb.nceas.metacat.DBUtil; import edu.ucsb.nceas.metacat.IdentifierManager; import edu.ucsb.nceas.metacat.admin.AdminException; import edu.ucsb.nceas.metacat.admin.upgrade.UpgradeUtilityInterface; import edu.ucsb.nceas.metacat.dataone.SystemMetadataFactory; import edu.ucsb.nceas.metacat.dataone.hazelcast.HazelcastService; import edu.ucsb.nceas.metacat.properties.PropertyService; import edu.ucsb.nceas.utilities.SortedProperties; public class GenerateSystemMetadata implements UpgradeUtilityInterface { private static Log log = LogFactory.getLog(GenerateSystemMetadata.class); private int serverLocation = 1; public boolean upgrade() throws AdminException { // do this in a thread too so that we don't have to hang the UI (web) ExecutorService executor = Executors.newSingleThreadExecutor(); Runnable command = new Runnable() { @Override public void run() { // just run it try { boolean success = multiThreadUpgrade(); } catch (AdminException e) { throw new RuntimeException(e); } } }; executor.execute(command); executor.shutdown(); // wait for it to finish before returning? boolean wait = false; if (wait) { log.debug("Waiting for upgrade to complete"); try { executor.awaitTermination(Long.MAX_VALUE, TimeUnit.HOURS); } catch (InterruptedException e) { AdminException ae = new AdminException(e.getMessage()); ae.initCause(e); throw ae; } log.debug("Done waiting for upgrade thread"); } return true; //return singleThreadUpgrade(); } /** * Use multiple threads to process parts of the complete ID list concurrently * @return * @throws AdminException */ public boolean multiThreadUpgrade() throws AdminException { boolean success = true; // do not include ORE or data, but can generate SystemMetadata for ALL records final boolean includeOre = false; final boolean downloadData = false; try { // the ids for which to generate system metadata List idList = null; // only get local objects idList = IdentifierManager.getInstance().getLocalIdsWithNoSystemMetadata(true, serverLocation); // for testing, subset to a limited random number boolean test = false; if (test) { //idList = DBUtil.getAllDocidsByType("eml://ecoinformatics.org/eml-2.1.0", true, serverLocation); idList = DBUtil.getAllDocids("knb-lter-gce"); // use a scope Collections.sort(idList); //Collections.shuffle(idList); int start = 0; int count = 100; int limit = Math.min(idList.size(), start + count); idList = idList.subList(start, limit); log.debug("limiting test list to: " + start + "-" + limit); for (String docid: idList) { log.debug("GENERATING SM TEST: " + docid); } } // make sure the list is sorted so we can break them into sublists for the threads Collections.sort(idList); // executor int availableProcessors = Runtime.getRuntime().availableProcessors(); int nThreads = availableProcessors * 1; //availableProcessors++; log.debug("Using nThreads: " + nThreads); ExecutorService executor = Executors.newFixedThreadPool(nThreads); int taskCount = 0; // init HZ log.debug("Making sure Hazelcast is up"); HazelcastService.getInstance(); // chunk into groups int fromIndex = 0; int toIndex = 0; String prefix = null; for (String docid: idList) { // increment the next entry, exclusive toIndex++; // use scope.docid (without revision) to determine groups // handle first document on its own, and a clause for the last document if (prefix == null || !docid.startsWith(prefix) || toIndex == idList.size()) { // construct a sublist for this previous group of docids final List subList = idList.subList(fromIndex, toIndex); log.debug("Grouping docid prefix: " + prefix); log.debug("subList.size: " + subList.size()); // add the task for this sublist Runnable command = new Runnable() { @Override public void run() { // generate based on this list try { log.debug("Processing subList.size: " + subList.size()); SystemMetadataFactory.generateSystemMetadata(subList, includeOre, downloadData); log.debug("Done processing subList.size: " + subList.size()); } catch (Exception e) { throw new RuntimeException(e); } } }; // execute the task executor.execute(command); taskCount++; // start at the end of this sublist fromIndex = toIndex; } log.debug("docid: " + docid); // get the previous docid prefix String previousId = docid; prefix = previousId.substring(0, previousId.lastIndexOf(".")); } log.info("done launching threaded tasks, count: " + taskCount); // wait for executor to finish executor.shutdown(); // wait a long time log.debug("Waiting for all threads to complete"); executor.awaitTermination(Long.MAX_VALUE, TimeUnit.HOURS); log.debug("Done waiting for all threads to complete"); // now we are ready to be a data one node PropertyService.setProperty("dataone.systemmetadata.generated", Boolean.TRUE.toString()); } catch (Exception e) { String msg = "Problem generating missing system metadata: " + e.getMessage(); log.error(msg, e); success = false; throw new AdminException(msg); } return success; } public int getServerLocation() { return serverLocation; } public void setServerLocation(int serverLocation) { this.serverLocation = serverLocation; } public static void main(String [] args){ try { // set up the properties based on the test/deployed configuration of the workspace SortedProperties testProperties = new SortedProperties("test/test.properties"); testProperties.load(); String metacatContextDir = testProperties.getProperty("metacat.contextDir"); PropertyService.getInstance(metacatContextDir + "/WEB-INF"); // make an upgrader instance GenerateSystemMetadata upgrader = new GenerateSystemMetadata(); // set any command line params, like the home server to run this for if (args.length > 0) { String serverLocation = args[0]; upgrader.setServerLocation(Integer.parseInt(serverLocation)); } // now run it upgrader.upgrade(); } catch (Exception ex) { System.out.println("Exception:" + ex.getMessage()); ex.printStackTrace(); } } }