Changes to support event values container

config/tests/generate_test_files.sh

@@ -16,15 +16,19 @@ fi
 
 rm -rf build/ dist/;
 
-./setup.py -q sdist_test_data;
+cp MANIFEST.test_data.in MANIFEST.in
+
+./setup.py -q sdist;
 
 if test $? -ne ${EXIT_SUCCESS};
 then
-	echo "Unable to run: ./setup.py sdist_test_data";
+	echo "Unable to run: ./setup.py sdist";
 
 	exit ${EXIT_FAILURE};
 fi
 
+git checkout MANIFEST.in
+
 SDIST_PACKAGE=`ls -1 dist/plaso-*.tar.gz | head -n1 | sed 's?^dist/??'`;
 
 if ! test "dist/${SDIST_PACKAGE}";
@@ -72,8 +76,8 @@ cp -rf ${SOURCE_DIRECTORY}/* .;
 TEST_FILE="psort_test.plaso";
 
 # Syslog does not contain a year we must pass preferred year to prevent the parser failing early on non-leap years.
-PYTHONPATH=. python ./tools/log2timeline.py --buffer_size=300 --quiet --preferred_year 2012 --storage-file ${TEST_FILE} test_data/syslog;
-PYTHONPATH=. python ./tools/log2timeline.py --quiet --timezone=Iceland --preferred_year 2012 --storage-file ${TEST_FILE} test_data/syslog;
+PYTHONPATH=. python ./plaso/scripts/log2timeline.py --buffer_size=300 --quiet --preferred_year 2012 --storage-file ${TEST_FILE} test_data/syslog/syslog;
+PYTHONPATH=. python ./plaso/scripts/log2timeline.py --quiet --timezone=Iceland --preferred_year 2012 --storage-file ${TEST_FILE} test_data/syslog/syslog;
 
 cat > tagging.txt <<EOI
 anacron1
@@ -86,7 +90,7 @@ repeated
   body contains 'last message repeated'
 EOI
 
-PYTHONPATH=. python ./tools/psort.py --analysis tagging --output-format=null --tagging-file=tagging.txt ${TEST_FILE};
+PYTHONPATH=. python ./plaso/scripts/psort.py --analysis tagging --output-format=null --tagging-file=tagging.txt ${TEST_FILE};
 
 # Run tagging twice.
 cat > tagging.txt <<EOI
@@ -100,13 +104,13 @@ repeated
   body contains 'last message repeated'
 EOI
 
-PYTHONPATH=. python ./tools/psort.py --analysis tagging --output-format=null --tagging-file=tagging.txt ${TEST_FILE};
+PYTHONPATH=. python ./plaso/scripts/psort.py --analysis tagging --output-format=null --tagging-file=tagging.txt ${TEST_FILE};
 
 mv ${TEST_FILE} ${OLD_PWD}/test_data/;
 
 TEST_FILE="pinfo_test.plaso";
 
-PYTHONPATH=. python ./tools/log2timeline.py --partition=all --quiet --storage-file ${TEST_FILE} test_data/tsk_volume_system.raw;
+PYTHONPATH=. python ./plaso/scripts/log2timeline.py --partition=all --quiet --storage-file ${TEST_FILE} test_data/tsk_volume_system.raw;
 
 mv ${TEST_FILE} ${OLD_PWD}/test_data/;
 

plaso/analysis/browser_search.py

@@ -271,7 +271,8 @@ def CompileReport(self, analysis_mediator):
     return super(BrowserSearchPlugin, self).CompileReport(analysis_mediator)
 
   def ExamineEvent(
-      self, analysis_mediator, event, event_data, event_data_stream):
+      self, analysis_mediator, event, event_data, event_data_stream,
+      event_values):
     """Analyzes an event.
 
     Args:
@@ -280,6 +281,7 @@ def ExamineEvent(
       event (EventObject): event.
       event_data (EventData): event data.
       event_data_stream (EventDataStream): event data stream.
+      event_values (AttributeContainer): event values attribute container.
     """
     if event_data.data_type not in self._SUPPORTED_EVENT_DATA_TYPES:
       return

plaso/analysis/chrome_extension.py

@@ -135,7 +135,8 @@ def CompileReport(self, analysis_mediator):
 
   # pylint: disable=unused-argument
   def ExamineEvent(
-      self, analysis_mediator, event, event_data, event_data_stream):
+      self, analysis_mediator, event, event_data, event_data_stream,
+      event_values):
     """Analyzes an event.
 
     Args:
@@ -144,6 +145,7 @@ def ExamineEvent(
       event (EventObject): event to examine.
       event_data (EventData): event data.
       event_data_stream (EventDataStream): event data stream.
+      event_values (AttributeContainer): event values attribute container.
     """
     if event_data.data_type not in self._SUPPORTED_EVENT_DATA_TYPES:
       return

plaso/analysis/hash_tagging.py

@@ -197,7 +197,8 @@ def CompileReport(self, analysis_mediator):
         analysis_mediator)
 
   def ExamineEvent(
-      self, analysis_mediator, event, event_data, event_data_stream):
+      self, analysis_mediator, event, event_data, event_data_stream,
+      event_values):
     """Evaluates whether an event contains the right data for a hash lookup.
 
     Args:
@@ -206,6 +207,7 @@ def ExamineEvent(
       event (EventObject): event.
       event_data (EventData): event data.
       event_data_stream (EventDataStream): event data stream.
+      event_values (AttributeContainer): event values attribute container.
     """
     if (not self._lookup_hash or not event_data_stream or
         event_data.data_type not in self.DATA_TYPES):

plaso/analysis/interface.py

@@ -86,7 +86,8 @@ def CompileReport(self, analysis_mediator):
 
   @abc.abstractmethod
   def ExamineEvent(
-      self, analysis_mediator, event, event_data, event_data_stream):
+      self, analysis_mediator, event, event_data, event_data_stream,
+      event_values):
     """Analyzes an event.
 
     Args:
@@ -95,4 +96,5 @@ def ExamineEvent(
       event (EventObject): event.
       event_data (EventData): event data.
       event_data_stream (EventDataStream): event data stream.
+      event_values (AttributeContainer): event values attribute container.
     """

plaso/analysis/sessionize.py

@@ -22,7 +22,8 @@ def __init__(self):
 
   # pylint: disable=unused-argument
   def ExamineEvent(
-      self, analysis_mediator, event, event_data, event_data_stream):
+      self, analysis_mediator, event, event_data, event_data_stream,
+      event_values):
     """Analyzes an EventObject and tags it as part of a session.
 
     Args:
@@ -31,6 +32,7 @@ def ExamineEvent(
       event (EventObject): event to examine.
       event_data (EventData): event data.
       event_data_stream (EventDataStream): event data stream.
+      event_values (AttributeContainer): event values attribute container.
     """
     if (self._session_end_timestamp is not None and
         event.timestamp > self._session_end_timestamp):

plaso/analysis/tagging.py

@@ -17,7 +17,8 @@ def __init__(self):
     self._tagging_rules = None
 
   def ExamineEvent(
-      self, analysis_mediator, event, event_data, event_data_stream):
+      self, analysis_mediator, event, event_data, event_data_stream,
+      event_values):
     """Labels events according to the rules in a tagging file.
 
     Args:
@@ -26,13 +27,15 @@ def ExamineEvent(
       event (EventObject): event to examine.
       event_data (EventData): event data.
       event_data_stream (EventDataStream): event data stream.
+      event_values (AttributeContainer): event values attribute container.
     """
     matched_label_names = []
-    for label_name, filter_objects in self._tagging_rules.items():
-      for filter_object in filter_objects:
-        # Note that tagging events based on existing labels is currently
-        # not supported.
-        if filter_object.Match(event, event_data, event_data_stream, None):
+    for label_name, event_filters in self._tagging_rules.items():
+      for event_filter in event_filters:
+        # Note that tagging events based on existing labels is currently not
+        # supported.
+        if event_filter.Match(
+            event, event_data, event_data_stream, event_values, None):
           matched_label_names.append(label_name)
           break
 

plaso/analysis/test_memory.py

@@ -35,7 +35,8 @@ def CompileReport(self, analysis_mediator):
 
   # pylint: disable=unused-argument
   def ExamineEvent(
-      self, analysis_mediator, event, event_data, event_data_stream):
+      self, analysis_mediator, event, event_data, event_data_stream,
+      event_values):
     """Analyzes an event.
 
     Args:
@@ -44,6 +45,7 @@ def ExamineEvent(
       event (EventObject): event.
       event_data (EventData): event data.
       event_data_stream (EventDataStream): event data stream.
+      event_values (AttributeContainer): event values attribute container.
     """
     self._objects.append(list(range(1024)))
 

plaso/analysis/unique_domains_visited.py

@@ -31,7 +31,8 @@ class UniqueDomainsVisitedPlugin(interface.AnalysisPlugin):
 
   # pylint: disable=unused-argument
   def ExamineEvent(
-      self, analysis_mediator, event, event_data, event_data_stream):
+      self, analysis_mediator, event, event_data, event_data_stream,
+      event_values):
     """Analyzes an event and extracts domains from it.
 
     We only evaluate straightforward web history events, not visits which can
@@ -43,6 +44,7 @@ def ExamineEvent(
       event (EventObject): event to examine.
       event_data (EventData): event data.
       event_data_stream (EventDataStream): event data stream.
+      event_values (AttributeContainer): event values attribute container.
     """
     if event_data.data_type not in self._SUPPORTED_EVENT_DATA_TYPES:
       return

plaso/containers/events.py

@@ -4,8 +4,8 @@
 import hashlib
 import re
 
-from acstore.containers import interface
-from acstore.containers import manager
+from acstore.containers import interface as containers_interface
+from acstore.containers import manager as containers_manager
 
 from dfdatetime import interface as dfdatetime_interface
 
@@ -28,8 +28,8 @@ def CalculateEventValuesHash(event_data, event_data_stream):
 
   for attribute_name, attribute_value in sorted(event_data.GetAttributes()):
     if attribute_value is None or attribute_name in (
-        '_event_data_stream_identifier', '_event_values_hash', '_parser_chain',
-        'data_type'):
+        '_event_data_stream_identifier', '_event_values_hash',
+        '_event_values_identifier', '_parser_chain', 'data_type'):
       continue
 
     # Ignore date and time values.
@@ -82,7 +82,7 @@ def CalculateEventValuesHash(event_data, event_data_stream):
   return md5_context.hexdigest()
 
 
-class DateLessLogHelper(interface.AttributeContainer):
+class DateLessLogHelper(containers_interface.AttributeContainer):
   """Attribute container to assist with logs without full dates.
 
   Attributes:
@@ -197,7 +197,7 @@ def SetEventDataStreamIdentifier(self, event_data_stream_identifier):
     self._event_data_stream_identifier = event_data_stream_identifier
 
 
-class EventData(interface.AttributeContainer):
+class EventData(containers_interface.AttributeContainer):
   """Event data attribute container.
 
   The event data attribute container represents the attributes of an entity,
@@ -212,6 +212,7 @@ class EventData(interface.AttributeContainer):
   _SERIALIZABLE_PROTECTED_ATTRIBUTES = [
       '_event_data_stream_identifier',
       '_event_values_hash',
+      '_event_values_identifier',
       '_parser_chain']
 
   def __init__(self, data_type=None):
@@ -223,6 +224,7 @@ def __init__(self, data_type=None):
     super(EventData, self).__init__()
     self._event_data_stream_identifier = None
     self._event_values_hash = None
+    self._event_values_identifier = None
     self._parser_chain = None
 
     self.data_type = data_type
@@ -280,8 +282,31 @@ def SetEventDataStreamIdentifier(self, event_data_stream_identifier):
     """
     self._event_data_stream_identifier = event_data_stream_identifier
 
+  def GetEventValuesIdentifier(self):
+    """Retrieves the identifier of the associated event values container.
 
-class EventDataStream(interface.AttributeContainer):
+    The event values identifier is a storage specific value that requires
+    special handling during serialization.
+
+    Returns:
+      AttributeContainerIdentifier: event values or None when not set.
+    """
+    return self._event_values_identifier
+
+  def SetEventValuesIdentifier(self, event_values_identifier):
+    """Sets the identifier of the associated event values container.
+
+    The event values identifier is a storage specific value that requires
+    special handling during serialization.
+
+    Args:
+      event_values_identifier (AttributeContainerIdentifier): event values
+          identifier.
+    """
+    self._event_values_identifier = event_values_identifier
+
+
+class EventDataStream(containers_interface.AttributeContainer):
   """Event data stream attribute container.
 
   The event data stream attribute container represents the attributes of
@@ -318,7 +343,7 @@ def __init__(self):
     self.yara_match = None
 
 
-class EventObject(interface.AttributeContainer):
+class EventObject(containers_interface.AttributeContainer):
   """Event attribute container.
 
   The framework is designed to parse files and create events
@@ -392,7 +417,7 @@ def SetEventDataIdentifier(self, event_data_identifier):
     self._event_data_identifier = event_data_identifier
 
 
-class EventTag(interface.AttributeContainer):
+class EventTag(containers_interface.AttributeContainer):
   """Event tag attribute container.
 
   Attributes:
@@ -501,7 +526,7 @@ def SetEventIdentifier(self, event_identifier):
 
 # TODO: the YearLessLogHelper attribute container is kept for backwards
 # compatibility remove once storage format 20230327 is obsolete.
-class YearLessLogHelper(interface.AttributeContainer):
+class YearLessLogHelper(containers_interface.AttributeContainer):
   """Year-less log helper attribute container.
 
   Attributes:
@@ -555,6 +580,6 @@ def SetEventDataStreamIdentifier(self, event_data_stream_identifier):
     self._event_data_stream_identifier = event_data_stream_identifier
 
 
-manager.AttributeContainersManager.RegisterAttributeContainers([
+containers_manager.AttributeContainersManager.RegisterAttributeContainers([
     DateLessLogHelper, EventData, EventDataStream, EventObject, EventTag,
     YearLessLogHelper])

plaso/filters/event_filter.py

@@ -30,13 +30,15 @@ def CompileFilter(self, filter_expression):
     self._event_filter = expression.Compile()
     self._filter_expression = filter_expression
 
-  def Match(self, event, event_data, event_data_stream, event_tag):
+  def Match(
+      self, event, event_data, event_data_stream, event_values, event_tag):
     """Determines if an event matches the filter.
 
     Args:
       event (EventObject): event.
       event_data (EventData): event data.
       event_data_stream (EventDataStream): event data stream.
+      event_values (AttributeContainer): event values attribute container.
       event_tag (EventTag): event tag.
 
     Returns:
@@ -46,4 +48,4 @@ def Match(self, event, event_data, event_data_stream, event_tag):
       return True
 
     return self._event_filter.Matches(
-        event, event_data, event_data_stream, event_tag)
+        event, event_data, event_data_stream, event_values, event_tag)