/** * '$RCSfile$' * Copyright: 2004 Regents of the University of California and the * National Center for Ecological Analysis and Synthesis * * '$Author$' * '$Date$' * '$Revision$' * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ package edu.ucsb.nceas.metacat; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Vector; import org.apache.log4j.Logger; import org.dataone.service.types.v1.Identifier; import org.dataone.service.types.v2.Log; import org.dataone.service.types.v2.LogEntry; import org.dataone.configuration.Settings; import org.dataone.service.types.v1.Event; import org.dataone.service.types.v1.NodeReference; import org.dataone.service.types.v1.Subject; import org.dataone.service.util.DateTimeMarshaller; import edu.ucsb.nceas.metacat.database.DBConnection; import edu.ucsb.nceas.metacat.database.DBConnectionPool; import edu.ucsb.nceas.metacat.database.DatabaseService; import edu.ucsb.nceas.metacat.index.MetacatSolrIndex; import edu.ucsb.nceas.metacat.properties.PropertyService; import edu.ucsb.nceas.metacat.util.DocumentUtil; import edu.ucsb.nceas.utilities.PropertyNotFoundException; /** * EventLog is used to intialize and store a log of events that occur in an * application. The events are registered with the logger as they occur, but * EventLog writes them to permenant storage when it is most convenient or * efficient. EventLog is a Singleton as there should always be only one object * for these logging events. * * TODO: Logging to the database needn't be synchronous with the event. * Instead, a separate thread can be launched that periodically sleeps and only * wakes periodically to see if metacat is idle. The log event can be cached * and inserted later when the thread wakes and finds metacat idle. * * TODO: Write a function that archives a part of the log table to an * external text file so that the log table doesn't get to big. This * function should be able to be called manually or on a schedule. * * TODO: Write an access function that returns an XML report for a * specific subset of events. Users should be able to query on * principal, docid/rev, date, event, and possibly other fields. * * @author jones */ public class EventLog { public static final String DELETE = "delete"; /** * The single instance of the event log that is always returned. */ private static EventLog self = null; private Logger logMetacat = Logger.getLogger(EventLog.class); private static final int USERAGENTLENGTH = 512; private EventLogFilter filter = null; private boolean enableEvenLogIndex= false; /** * A private constructor that initializes the class when getInstance() is * called. */ private EventLog() { filter = new EventLogFilter(); enableEvenLogIndex = Settings.getConfiguration().getBoolean("index.accessLog.count.enabled", false); } /** * Return the single instance of the event log after initializing it if it * wasn't previously initialized. * * @return the single EventLog instance */ public static EventLog getInstance() { if (self == null) { self = new EventLog(); } return self; } /** * Log an event of interest to the application. The information logged can * include basic identification information about the principal or computer * that initiated the event. * * @param ipAddress the internet protocol address for the event * @param userAgent the agent making the request * @param principal the principal for the event (a username, etc) * @param docid the identifier of the document to which the event applies * @param event the string code for the event */ public void log(String ipAddress, String userAgent, String principal, String docid, String event) { EventLogData logData = new EventLogData(ipAddress, userAgent, principal, docid, event); boolean filterOut = false; if(filter != null) { filterOut = filter.filter(logData); } if(!filterOut) { insertLogEntry(logData); // update the event information in the index try { String localId = DocumentUtil.getSmartDocId(docid); int rev = DocumentUtil.getRevisionFromAccessionNumber(docid); String guid = IdentifierManager.getInstance().getGUID(localId, rev); Identifier pid = new Identifier(); pid.setValue(guid); // submit for indexing if(enableEvenLogIndex) { MetacatSolrIndex.getInstance().submit(pid, null, this.getIndexFields(pid, event), false); } } catch (Exception e) { logMetacat.error("Could not update event index information", e); } } else { logMetacat.warn("EventLog.log - The event - "+event+" generated by the user "+principal+" at the address "+ipAddress+ " is in the blacklist and Metacat wouldn't log it."); } } public Map> getIndexFields(Identifier pid, String event) { // update the search index for the event try { if (event != null) { String fieldName = event + "_count_i"; int eventCount = 0; String docid = IdentifierManager.getInstance().getLocalId(pid.getValue()); Log eventLog = this.getD1Report(null, null, new String[] {docid}, event, null, null, false, 0, 0); eventCount = eventLog.getTotal(); List values = new ArrayList(); values.add(eventCount); Map> fields = new HashMap>(); fields.put(fieldName, values); return fields; } } catch (Exception e) { logMetacat.error("Could not update event index information on pid: " + pid.getValue() + " for event: " + event, e); } // default if we can't find the event information return null; } /** * Insert a single log event record to the database. * * @param logData the data to be logged when an event occurs */ private void insertLogEntry(EventLogData logData) { String insertString = "insert into access_log" + "(ip_address, user_agent, principal, docid, " + "event, date_logged) " + "values ( ?, ?, ?, ?, ?, ? )"; DBConnection dbConn = null; int serialNumber = -1; try { // Get a database connection from the pool dbConn = DBConnectionPool.getDBConnection("EventLog.insertLogEntry"); serialNumber = dbConn.getCheckOutSerialNumber(); String userAgent = logData.getUserAgent(); if(userAgent != null && userAgent.length() > USERAGENTLENGTH) { userAgent = userAgent.substring(0, USERAGENTLENGTH); } // Execute the insert statement PreparedStatement stmt = dbConn.prepareStatement(insertString); stmt.setString(1, logData.getIpAddress()); stmt.setString(2, userAgent); stmt.setString(3, logData.getPrincipal()); stmt.setString(4, logData.getDocid()); stmt.setString(5, logData.getEvent()); stmt.setTimestamp(6, new Timestamp(new Date().getTime())); stmt.executeUpdate(); stmt.close(); } catch (SQLException e) { logMetacat.error("Error while logging event to database: " + e.getMessage()); } finally { // Return database connection to the pool DBConnectionPool.returnDBConnection(dbConn, serialNumber); } } /** * Get a report of the log events that match a set of filters. The * filter parameters can be null; log records are subset based on * non-null filter parameters. * * @param ipAddress the internet protocol address for the event * @param principal the principal for the event (a username, etc) * @param docid the identifier of the document to which the event applies * @param event the string code for the event * @param startDate beginning of date range for query * @param endDate end of date range for query * @return an XML-formatted report of the access log entries */ public String getReport(String[] ipAddress, String[] principal, String[] docid, String[] event, Timestamp startDate, Timestamp endDate, boolean anonymous) { StringBuffer resultDoc = new StringBuffer(); StringBuffer query = new StringBuffer(); query.append("select entryid, ip_address, user_agent, principal, docid, " + "event, date_logged from access_log"); // + "" // + "event, date_logged) " + "values (" + "'" // + logData.getIpAddress() + "', " + "'" // + logData.getPrincipal() + "', " + "'" // + logData.getDocid() + "', " + "'" + logData.getEvent() // + "', " + " ? " + ")"; if (ipAddress != null || principal != null || docid != null || event != null || startDate != null || endDate != null) { query.append(" where "); } boolean clauseAdded = false; int startIndex = 0; int endIndex = 0; List paramValues = new ArrayList(); if (ipAddress != null) { query.append("ip_address in ("); for (int i = 0; i < ipAddress.length; i++) { if (i > 0) { query.append(", "); } query.append("?"); paramValues.add(ipAddress[i]); } query.append(") "); clauseAdded = true; } if (principal != null) { if (clauseAdded) { query.append(" and "); } query.append("principal in ("); for (int i = 0; i < principal.length; i++) { if (i > 0) { query.append(", "); } query.append("?"); paramValues.add(principal[i]); } query.append(") "); clauseAdded = true; } if (docid != null) { if (clauseAdded) { query.append(" and "); } query.append("docid in ("); for (int i = 0; i < docid.length; i++) { if (i > 0) { query.append(", "); } query.append("?"); String fullDocid = docid[i]; // allow docid without revision - look up latest version try { fullDocid = DocumentUtil.appendRev(fullDocid); } catch (Exception e) { // just warn about this logMetacat.debug("Could not check docid for revision: " + fullDocid, e); } paramValues.add(fullDocid); } query.append(") "); clauseAdded = true; } if (event != null) { if (clauseAdded) { query.append(" and "); } query.append("event in ("); for (int i = 0; i < event.length; i++) { if (i > 0) { query.append(", "); } query.append("?"); paramValues.add(event[i]); } query.append(") "); clauseAdded = true; } if (startDate != null) { if (clauseAdded) { query.append(" and "); } query.append("date_logged >= ?"); clauseAdded = true; startIndex++; } if (endDate != null) { if (clauseAdded) { query.append(" and "); } query.append("date_logged < ?"); clauseAdded = true; endIndex = startIndex + 1; } DBConnection dbConn = null; int serialNumber = -1; try { // Get a database connection from the pool dbConn = DBConnectionPool.getDBConnection("EventLog.getReport"); serialNumber = dbConn.getCheckOutSerialNumber(); // Execute the query statement PreparedStatement stmt = dbConn.prepareStatement(query.toString()); //set the param values int parameterIndex = 1; for (String val: paramValues) { stmt.setString(parameterIndex++, val); } if (startDate != null) { stmt.setTimestamp(parameterIndex++, startDate); } if (endDate != null) { stmt.setTimestamp(parameterIndex++, endDate); } stmt.execute(); ResultSet rs = stmt.getResultSet(); //process the result and return it as an XML document resultDoc.append("\n"); resultDoc.append("\n"); while (rs.next()) { resultDoc.append( generateXmlRecord( rs.getString(1), //id anonymous ? "" : rs.getString(2), //ip rs.getString(3), //userAgent anonymous ? "" : rs.getString(4), //principal rs.getString(5), rs.getString(6), rs.getTimestamp(7))); } resultDoc.append(""); stmt.close(); } catch (SQLException e) { logMetacat.info("Error while logging event to database: " + e.getMessage()); } finally { // Return database connection to the pool DBConnectionPool.returnDBConnection(dbConn, serialNumber); } return resultDoc.toString(); } /** * A utility method to determine if the given docid was deleted. * @param docid the specified docid * @return true if there is a delete event for the id; false otherwise. */ public boolean isDeleted(String docid) { boolean deleted =false; if(docid != null || !docid.trim().equals("")) { String[] docids = new String[1]; docids[0] = docid; String[] events = new String[1]; events[0]= DELETE; String[] ipAddress = null; String[] principal = null; Timestamp startDate = null; Timestamp endDate = null; boolean anonymous = false; String report =getReport(ipAddress, principal, docids, events, startDate, endDate, anonymous); //System.out.println("the report is "+report); if(report != null && report.contains(""+DELETE+"") ){ deleted = true; } } return deleted; } public Log getD1Report(String[] ipAddress, String[] principal, String[] docid, String event, Timestamp startDate, Timestamp endDate, boolean anonymous, Integer start, Integer count) { boolean isCreateEvent = false; Log log = new Log(); NodeReference memberNode = new NodeReference(); String nodeId = "localhost"; try { nodeId = PropertyService.getProperty("dataone.nodeId"); } catch (PropertyNotFoundException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } memberNode.setValue(nodeId); // subquery does the heavy lifting StringBuffer subQueryFrom = new StringBuffer(); subQueryFrom.append("from access_log "); boolean clauseAdded = false; List paramValues = new ArrayList(); if (ipAddress != null) { if (clauseAdded) { subQueryFrom.append(" and "); } else { subQueryFrom.append(" where "); } subQueryFrom.append("ip_address in ("); for (int i = 0; i < ipAddress.length; i++) { if (i > 0) { subQueryFrom.append(", "); } subQueryFrom.append("?"); paramValues.add(ipAddress[i]); } subQueryFrom.append(") "); clauseAdded = true; } if (principal != null) { if (clauseAdded) { subQueryFrom.append(" and "); } else { subQueryFrom.append(" where "); } subQueryFrom.append("principal in ("); for (int i = 0; i < principal.length; i++) { if (i > 0) { subQueryFrom.append(", "); } subQueryFrom.append("?"); paramValues.add(principal[i]); } subQueryFrom.append(") "); clauseAdded = true; } if (docid != null) { if (clauseAdded) { subQueryFrom.append(" and "); } else { subQueryFrom.append(" where "); } subQueryFrom.append("docid in ("); for (int i = 0; i < docid.length; i++) { if (i > 0) { subQueryFrom.append(", "); } subQueryFrom.append("?"); paramValues.add(docid[i]); } subQueryFrom.append(") "); clauseAdded = true; } //please make sure the handling of event is just before the the startDate clause!!! if (event != null) { if (clauseAdded) { subQueryFrom.append(" and "); } else { subQueryFrom.append(" where "); } subQueryFrom.append("event in ("); subQueryFrom.append("?"); String eventString = event; if (eventString.equals(Event.CREATE.xmlValue())) { isCreateEvent = true; //since the create event maps create, insert and et al, we handle it in different way subQueryFrom.append(",?");// for INSERT, the insert is handled by line 508 subQueryFrom.append(",?");// for upload subQueryFrom.append(",?");// for UPLOAD subQueryFrom.append(",?");// for create //eventString = "insert,INSERT,upload,UPLOAD,create"; } else { paramValues.add(eventString); } subQueryFrom.append(") "); clauseAdded = true; } if (startDate != null) { if (clauseAdded) { subQueryFrom.append(" and "); } else { subQueryFrom.append(" where "); } subQueryFrom.append("date_logged >= ?"); clauseAdded = true; } if (endDate != null) { if (clauseAdded) { subQueryFrom.append(" and "); } else { subQueryFrom.append(" where "); } subQueryFrom.append("date_logged < ?"); clauseAdded = true; } // count query String countSelect = "select count(*) "; // subquery select String subquerySelect = "select entryid "; // for selecting fields we want in the join String fieldSelect = "select " + "entryid, " + "id.guid as identifier, " + "ip_address, " + "user_agent, " + "principal, " + "case " + " when event = 'insert' then 'create' " + " when event = 'INSERT' then 'create' " + " when event = 'upload' then 'create' " + " when event = 'UPLOAD' then 'create' " + " else event " + "end as event, " + "date_logged " + "from access_log al, identifier id " + "where al.docid = id.docid||'.'||id.rev " + "and al.entryid in "; // order by String orderByClause = " order by entryid "; // select the count String countQuery = countSelect + subQueryFrom.toString(); logMetacat.debug("The count query is " + countQuery); // select the fields using paged subquery and fields join query String pagedSubquery = DatabaseService.getInstance().getDBAdapter().getPagedQuery(subquerySelect + subQueryFrom.toString() + orderByClause, start, count); String pagedQuery = fieldSelect + " ( " + pagedSubquery + " ) " + orderByClause; logMetacat.debug("The selection query is " + pagedQuery); logMetacat.debug("The startDate in the query is " + startDate); logMetacat.debug("The endDate in the query is " + startDate); DBConnection dbConn = null; int serialNumber = -1; try { // Get a database connection from the pool dbConn = DBConnectionPool.getDBConnection("EventLog.getD1Report"); serialNumber = dbConn.getCheckOutSerialNumber(); // Execute the query statement PreparedStatement fieldsStmt = dbConn.prepareStatement(pagedQuery); PreparedStatement countStmt = dbConn.prepareStatement(countQuery); //set the param values int parameterIndex = 1; for (String val: paramValues) { countStmt.setString(parameterIndex, val); fieldsStmt.setString(parameterIndex, val); parameterIndex++; } if(isCreateEvent) { //handle the event mapping. If we add another event mapping, we need add a "?" in line 496 //those values are not in the paramValues. countStmt.setString(parameterIndex, "insert"); fieldsStmt.setString(parameterIndex, "insert"); parameterIndex++; countStmt.setString(parameterIndex, "INSERT"); fieldsStmt.setString(parameterIndex, "INSERT"); parameterIndex++; countStmt.setString(parameterIndex, "upload"); fieldsStmt.setString(parameterIndex, "upload"); parameterIndex++; countStmt.setString(parameterIndex, "UPLOAD"); fieldsStmt.setString(parameterIndex, "UPLOAD"); parameterIndex++; countStmt.setString(parameterIndex, "create"); fieldsStmt.setString(parameterIndex, "create"); parameterIndex++; } if (startDate != null) { countStmt.setTimestamp(parameterIndex, startDate); fieldsStmt.setTimestamp(parameterIndex, startDate); parameterIndex++; } if (endDate != null) { countStmt.setTimestamp(parameterIndex, endDate); fieldsStmt.setTimestamp(parameterIndex, endDate); parameterIndex++; } // for the return Log list List logs = new Vector(); // get the fields form the query if (count != 0) { long startTime = System.currentTimeMillis(); logMetacat.debug("Time to start to execute the selection query "+startTime); fieldsStmt.execute(); long endTime = System.currentTimeMillis(); logMetacat.debug("Time to run the selection query is "+(endTime-startTime)/1000+" seconds."); ResultSet rs = fieldsStmt.getResultSet(); //process the result and return it while (rs.next()) { LogEntry logEntry = new LogEntry(); logEntry.setEntryId(rs.getString(1)); Identifier identifier = new Identifier(); identifier.setValue(rs.getString(2)); logEntry.setIdentifier(identifier); logEntry.setIpAddress(anonymous ? "N/A" : rs.getString(3)); String userAgent = "N/A"; if (rs.getString(4) != null) { userAgent = rs.getString(4); } logEntry.setUserAgent(userAgent); Subject subject = new Subject(); subject.setValue(anonymous ? "N/A" : rs.getString(5)); logEntry.setSubject(subject); String logEventString = rs.getString(6); if(logEventString == null) { logEventString = "unknown"; } logEntry.setEvent(logEventString); logEntry.setDateLogged(rs.getTimestamp(7)); logEntry.setNodeIdentifier(memberNode); logs.add(logEntry); } fieldsStmt.close(); long endTime2 = System.currentTimeMillis(); logMetacat.debug("Time to put the query result to the log is "+(endTime2-endTime)/1000+" seconds."); } // set what we have log.setLogEntryList(logs); log.setStart(start); log.setCount(logs.size()); // get total for out query int total = 0; long startTime = System.currentTimeMillis(); logMetacat.debug("Time to start to execute the counting query "+startTime); countStmt.execute(); long endTime = System.currentTimeMillis(); logMetacat.debug("Time to run the counting query is "+(endTime-startTime)/1000+" seconds."); ResultSet countRs = countStmt.getResultSet(); if (countRs.next()) { total = countRs.getInt(1); } countStmt.close(); log.setTotal(total); } catch (SQLException e) { logMetacat.error("Error while getting log events: " + e.getMessage(), e); } finally { // Return database connection to the pool DBConnectionPool.returnDBConnection(dbConn, serialNumber); } return log; } /** * Format each returned log record as an XML structure. * * @param entryId the identifier of the log entry * @param ipAddress the internet protocol address for the event * @param the agent making the request * @param principal the principal for the event (a username, etc) * @param docid the identifier of the document to which the event applies * @param event the string code for the event * @param dateLogged the date on which the event occurred * @return String containing the formatted XML */ private String generateXmlRecord(String entryId, String ipAddress, String userAgent, String principal, String docid, String event, Timestamp dateLogged) { StringBuffer rec = new StringBuffer(); rec.append(""); rec.append(generateXmlElement("entryid", entryId)); rec.append(generateXmlElement("ipAddress", ipAddress)); rec.append(generateXmlElement("userAgent", userAgent)); rec.append(generateXmlElement("principal", principal)); rec.append(generateXmlElement("docid", docid)); rec.append(generateXmlElement("event", event)); rec.append(generateXmlElement("dateLogged", DateTimeMarshaller.serializeDateToUTC(dateLogged))); rec.append("\n"); return rec.toString(); } /** * Return an XML formatted element for a given name/value pair. * * @param name the name of the xml element * @param value the content of the xml element * @return the formatted XML element as a String */ private String generateXmlElement(String name, String value) { return "<" + name + ">" + value + ""; } }