Merge branch 'mboman-master'

docker/Dockerfile

@@ -0,0 +1,47 @@
+#
+# This Docker image encapsulates Maltrieve, a tool to retrieve malware
+# directly from the source for security researchers.
+# which was created by Kyle Maxwell (technoskald) and is
+# available at https://github.com/technoskald/maltrieve.
+#
+# The file below is based on ideas from Spenser Reinhardt's Dockerfile
+# (https://registry.hub.docker.com/u/sreinhardt/honeynet/dockerfile)
+# and on instructions outlined by M. Fields (@shakey_1).
+#
+# To run this image after installing Docker, use a command like this:
+#
+# sudo docker run --rm -it technoskald/maltrieve bash
+#
+# then run ./maltrieve.py with the desired parameters.
+
+FROM ubuntu:14.04
+MAINTAINER Michael Boman <michael@michaelboman.org>
+
+USER root
+RUN apt-get update && \
+  apt-get install -y --no-install-recommends \
+    gcc \
+    git \
+    libpython2.7-stdlib \
+    python2.7 \
+    python2.7-dev \
+    python-pip \
+    python-setuptools && \
+
+  rm -rf /var/lib/apt/lists/* && \
+
+  groupadd -r maltrieve && \
+  useradd -r -g maltrieve -d /home/maltrieve -s /sbin/nologin -c "Maltrieve User" maltrieve
+
+WORKDIR /home
+RUN git clone https://github.com/technoskald/maltrieve.git && \
+  cd maltrieve && \
+  pip install -r requirements.txt && \
+  chown -R maltrieve:maltrieve /home/maltrieve
+
+USER maltrieve
+ENV HOME /home/maltrieve
+ENV USER maltrieve
+WORKDIR /home/maltrieve
+CMD ["./maltrieve.py"]
+

maltrieve.py

@@ -123,7 +123,15 @@ def save_malware(response, directory, black_list, white_list):
         stored = True
     # else save to disk
     if not stored:
-        with open(os.path.join(directory, md5), 'wb') as f:
+        if cfg['sort_mime']:
+            # set folder per mime_type
+            sort_folder =mime_type.replace('/', '_')
+            if not os.path.exists(os.path.join(directory, sort_folder)):
+                os.makedirs(os.path.join(directory, sort_folder))
+            store_path = os.path.join(directory, sort_folder, md5)
+        else:
+            store_path = os.path.join(directory, md5)
+        with open(store_path, 'wb') as f:
             f.write(data)
             logging.info("Saved %s to dump dir" % md5)
     return True
@@ -194,7 +202,10 @@ def main():
                         action="store_true", default=False)
     parser.add_argument("-c", "--cuckoo",
                         help="Enable cuckoo analysis", action="store_true", default=False)
-
+    parser.add_argument("-s", "--sort_mime",
+                        help="Sort Files By Mime", action="store_true", default=False)
+
+
     global cfg
     cfg = dict()
     args = parser.parse_args()
@@ -227,6 +238,8 @@ def main():
         cfg['User-Agent'] = {'User-Agent': config.get('Maltrieve', 'User-Agent')}
     else:
         cfg['User-Agent'] = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)"
+
+    cfg['sort_mime'] = args.sort_mime
 
     if cfg['proxy']:
         logging.info('Using proxy %s', cfg['proxy'])
@@ -285,7 +298,8 @@ def main():
 
     print "Processing source URLs"
 
-    source_urls = {'http://www.malwaredomainlist.com/hostslist/mdl.xml': process_xml_list_desc,
+    source_urls = {"https://zeustracker.abuse.ch/monitor.php?urlfeed=binaries":process_xml_list_desc,
+        'http://www.malwaredomainlist.com/hostslist/mdl.xml': process_xml_list_desc,
                    'http://malc0de.com/rss/': process_xml_list_desc,
                    # 'http://www.malwareblacklist.com/mbl.xml',   # removed for now
                    'http://vxvault.siri-urz.net/URL_List.php': process_simple_list,