diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d7d7182 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3 + +WORKDIR /app + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY imap2thehive.py ./imap2thehive.py + +CMD [ "python", "imap2thehive.py" ] diff --git a/Dockerfile.CI b/Dockerfile.CI new file mode 100644 index 0000000..3349fe8 --- /dev/null +++ b/Dockerfile.CI @@ -0,0 +1,10 @@ +FROM python:3 + +WORKDIR /app + +COPY git_src/requirements.txt /app/requirements.txt +COPY git_src/imap2thehive.py /app/imap2thehive.py + +RUN pip install --no-cache-dir -r /app/requirements.txt + +CMD [ "python", "imap2thehive.py" ] diff --git a/README.md b/README.md index ff79e53..8023260 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,20 @@ -Miscelaneous Docker files for my personal usage. -Feel free to contribute. +# imap2thehive + +imap2thehive image with analyzers installed + +image built around: https://github.com/xme/dockers/tree/master/imap2thehive + +Image name: `pipedrive/imap2thehive` + +# Description +This image is a K8 ready version of imap2thehive. The Python script included in the image diviates slightly from the original version of XME. + + +# Live deployment +1. [Deploy via Jenkins][1] +2. After deploying, fetch the new tag that was pushed to Docker Hub. This is listed in the console output of the Jenkins job. +3. Update the tag in the K8 manifests in [chef-repo][2] and in [the-hive-kube][3] + +[1]: https://jenkins.pipedrive.tools/job/baseimage-imap2thehive-docker/ +[2]: https://github.com/pipedrive/chef-repo +[3]: https://github.com/pipedrive/the-hive-kube \ No newline at end of file diff --git a/cvesearch/Dockerfile b/cvesearch/Dockerfile deleted file mode 100644 index 001d87f..0000000 --- a/cvesearch/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM ubuntu -MAINTAINER Matt Erasmus -RUN apt-get update -RUN apt-get install -yq git python3-pip python3-pymongo mongodb libxml2-dev python3-lxml redis-server tmux -RUN git clone https://github.com/cve-search/cve-search.git /opt/cve-search -WORKDIR /opt/cve-search -RUN pip3 install -r requirements.txt -WORKDIR /opt/cve-search/etc -RUN cp configuration.ini.sample configuration.ini -RUN sed -i 's/Host: 127.0.0.1/Host: 0.0.0.0/' configuration.ini -WORKDIR /root -COPY init.sh /root/init.sh -RUN chmod 700 /root/init.sh -RUN touch /.firstboot -EXPOSE 5000 -CMD ["/root/init.sh"] \ No newline at end of file diff --git a/cvesearch/README.md b/cvesearch/README.md deleted file mode 100644 index edcac1d..0000000 --- a/cvesearch/README.md +++ /dev/null @@ -1,17 +0,0 @@ -Introduction -============ -Docker file to run cve-search (https://github.com/cve-search/cve-search) -Based on https://github.com/znb/Docker/tree/master/CVE-Search -The web interface is binding to 0.0.0.0:5000 - -Build -===== -$ docker built -t cvesearch:latest Dockerfile . - -Usage -===== -$ docker run cvesearch:latest -p 5000:5000 - -Or use the docker compose file: - -$ docker-compose run diff --git a/cvesearch/docker-compose.yml b/cvesearch/docker-compose.yml deleted file mode 100644 index 9a39149..0000000 --- a/cvesearch/docker-compose.yml +++ /dev/null @@ -1,15 +0,0 @@ -# -# Docker compose file for CVE-Search -# -version: '2' -networks: - bridge: - driver: bridge -services: - cve-search: - image: "xme/cvesearch" - container_name: cvesearch - ports: - - "5000:5000" - networks: - - bridge diff --git a/cvesearch/init.sh b/cvesearch/init.sh deleted file mode 100644 index ec4b459..0000000 --- a/cvesearch/init.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -CVESEARCHDIR=/opt/cve-search - -echo "Starting services..." -echo -service mongodb start -service redis-server start -echo -if [ -r /.firstboot ]; then - echo "Initializing Databases... (Please note this will take a while)" - echo - cd ${CVESEARCHDIR}/sbin - ./db_mgmt.py -p - ./db_mgmt_cpe_dictionary.py - ./db_updater.py -c - rm /.firstboot -fi -echo -python3 /opt/cve-search/web/index.py \ No newline at end of file diff --git a/dssuite/Dockerfile b/dssuite/Dockerfile deleted file mode 100644 index 1649330..0000000 --- a/dssuite/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -# -# Dockerfile to build a "DSS" container (Didier Stevens's Suite) -# -# @author: Xavier Mertens -# @copyright: AGPLv3 (http://www.gnu.org/licenses/agpl.html) -# - -FROM ubuntu:16.04 -MAINTAINER Xavier Mertens - -ENV VERSION 1.1 - -RUN apt-get update && \ - apt-get upgrade -y && \ - apt-get install -y python-pip git figlet && \ - rm -rf /var/lib/apt/lists/* - -RUN git clone https://github.com/DidierStevens/DidierStevensSuite /usr/local/bin -RUN chmod a+x /usr/local/bin/* -RUN pip install olefile - -RUN echo "#!/bin/bash" >/entrypoint.sh -RUN echo "test \"\$1\" == \"/bin/bash\" && figlet \"DSSuite\" && echo Version $VERSION - Help: https://blog.didierstevens.com/my-software/ && echo" >>/entrypoint.sh -RUN echo 'exec $@' >>/entrypoint.sh -RUN chmod a+x /entrypoint.sh - -VOLUME ["/malware"] -WORKDIR /malware - -ENTRYPOINT ["/entrypoint.sh"] -CMD ["/bin/bash"] diff --git a/imap2thehive/imap2thehive.py b/imap2thehive.py similarity index 98% rename from imap2thehive/imap2thehive.py rename to imap2thehive.py index f4dbf5f..fdcbf73 100755 --- a/imap2thehive/imap2thehive.py +++ b/imap2thehive.py @@ -44,7 +44,7 @@ __author__ = "Xavier Mertens" __license__ = "GPLv3" -__version__ = "1.0.7" +__version__ = "1.0.8" __maintainer__ = "Xavier Mertens" __email__ = "xavier@rootshell.be" __name__ = "imap2thehive" @@ -138,7 +138,8 @@ def searchObservables(buffer, observables): { 'type': 'mail', 'regex': r'\b([a-z][_a-z0-9-.+]+@[a-z0-9-.]+\.[a-z]+)\b' }, { 'type': 'hash', 'regex': r'\b([a-f0-9]{32}|[A-F0-9]{32})\b' }, { 'type': 'hash', 'regex': r'\b([a-f0-9]{40}|[A-F0-9]{40})\b' }, - { 'type': 'hash', 'regex': r'\b([a-f0-9]{64}|[A-F0-9]{64})\b' } + { 'type': 'hash', 'regex': r'\b([a-f0-9]{64}|[A-F0-9]{64})\b' }, + { 'type': 'headers', 'regex': r'\b(([A-Za-z-]+):\s([^\r\n]+(?:\r?\n(?![A-Za-z-]+:\s)[^\r\n]+)*))\b'} ] # Add custom observables if any @@ -216,8 +217,8 @@ def submitTheHive(message): while i < len(headers.keys()): headers_string = headers_string + headers.keys()[i] + ': ' + headers.values()[i] + '\n' i+=1 - # Temporary disabled - # observables = searchObservables(headers_string, observables) + # Testing email headers. + observables.extend(searchObservables(headers_string, observables)) body = '' for part in msg.walk(): @@ -545,5 +546,10 @@ def main(): return if __name__ == 'imap2thehive': - main() - sys.exit(0) + log = logging.getLogger(__name__) + + while True: + log.info('Running main loop') + main() + log.info('Finished main loop, sleeping') + time.sleep(120) diff --git a/imap2thehive/Dockerfile b/imap2thehive/Dockerfile deleted file mode 100644 index 83ede03..0000000 --- a/imap2thehive/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM ubuntu:16.04 -MAINTAINER Xavier Mertens - -RUN apt-get update && \ - apt-get install -y python3-pip cron logrotate && \ - rm -rf /var/lib/apt/lists/* - -WORKDIR /tmp -COPY requirements.txt . -RUN pip3 install -r requirements.txt - -WORKDIR /opt -COPY imap2thehive.py . - -# Create the cronjob -# An empty line is mandatory! -RUN echo '*/5 * * * * root (/opt/imap2thehive.py -c /etc/imap2thehive.conf -v >>/var/log/cron.log 2>&1)' >>/etc/crontab -RUN echo '' >>/etc/crontab -RUN touch /var/log/cron.log -RUN touch /.firstboot -COPY entrypoint.sh . -RUN chmod 0755 /opt/entrypoint.sh - -ENTRYPOINT [ "/opt/entrypoint.sh" ] diff --git a/imap2thehive/README.md b/imap2thehive/README.md deleted file mode 100644 index 1f098fc..0000000 --- a/imap2thehive/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Purpose -The script imap2thehive.py polls an IMAP4 mailbox for new emails and imports fetched messages into an instance of [TheHive](https://thehive-project.org/). By default, a new case is created per email read. If the subject of the mail contains "[ALERT]", an alert is created. - -# Configuration -The script is fully configurable via a Python-friendly configuration file. See imap2thehive.conf sample for more details. - -# Usage -The script can be run manually to import a mailbox or it can be scheduled to run at fixed interval with a cron job. The syntax is simple: -``` -# ./imap2thehive.py -h -usage: imap2thehive.py [-h] [-v] [-c CONFIG] - -Process an IMAP folder to create TheHive alerts/cased. - -optional arguments: - -h, --help show this help message and exit - -v, --verbose verbose output - -c CONFIG, --config CONFIG - configuration file (default: /etc/imap2thehive.conf) -``` - -# Docker Container -I created a Dockerfile to build a container: - -``` -# git clone https://github.com/xme/dockers -# cd imap2thehive -# docker build -t imap2thehive:latest . -# docker run -v $PWD/imap2thehive.conf:/etc/imap2thehive.conf:ro imap2thehive -``` - -# Observables Whitelisting -The script is able to extract observables (emails, URLs, files, hashes). To avoid too many false positives, it is possible to create whitelists (based on regular expressions). See the file imap2thehive.whitelists. \ No newline at end of file diff --git a/imap2thehive/entrypoint.sh b/imap2thehive/entrypoint.sh deleted file mode 100644 index acf86b8..0000000 --- a/imap2thehive/entrypoint.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -set -e -if [ -r /.firstboot ]; then - # Enable logrotate - cat <<__LOGROTATE__ >/etc/cron.daily/logrotate -#!/bin/sh -test -x /usr/sbin/logrotate || exit 0 -/usr/sbin/logrotate /opt/logrotate.conf -__LOGROTATE__ - - cat <<__LOGROTATE2__ >/opt/logrotate.conf -/var/log/cron.log -{ - rotate 7 - daily - missingok - notifempty - delaycompress - compress -} -__LOGROTATE2__ - chmod 0755 /etc/cron.daily/logrotate - rm /.firstboot -fi - -if [ ! -r /etc/imap2thehive.conf ]; then - echo "[ERROR] Cannot read /etc/imap2thehive.conf. Volume not defined?" - exit 1 -fi -echo "Imap2TheHive polling started..." -/usr/sbin/cron && tail -F /var/log/cron.log diff --git a/imap2thehive/imap2thehive.conf b/imap2thehive/imap2thehive.conf deleted file mode 100644 index de8c5bc..0000000 --- a/imap2thehive/imap2thehive.conf +++ /dev/null @@ -1,69 +0,0 @@ -[imap] -host: your.mail.server -port: 993 -user: your_user -password: your_password -folder: incoming -expunge: true -spam: (X-Spam-Flag: YES) - -[thehive] -url: your_thehive_instance -apikey: thehive_user_key -observables: true -whitelists: imap2thehive.whitelists - -[alert] -tlp: 3 -tags: email -keywords: \S*(ALERT|VTMIS)\S* - -[case] -tlp: 3 -tags: email -tasks: Investigation,Communication,Tracking -template: thehive_template -files: application/pdf - -[custom_observables] -ossec_asset: Received From: \((\w+)\)\s -ossec_level: Rule: \w+ fired \(level (\d+)\)\s- -ossec_message: Rule: \w+ fired \(level \d+\)\s-> "(.*)" -ossec_rule: Rule: (\d+) fired \(level - -[loggers] -keys=root,imap2thehive - -[handlers] -keys=console,file - -[formatters] -keys=simple,console - -[logger_root] -level=INFO -handlers=console - -[logger_imap2thehive] -level=DEBUG -handlers=file -propagate=1 -qualname=imap2thehive - -[handler_console] -class=StreamHandler -formatter=console -level=DEBUG -args=(sys.stdout,) - -[handler_file] -class=FileHandler -level=WARNING -formatter=simple -args=('/var/log/imap2thehive.log', 'w') - -[formatter_simple] -format=%(asctime)s %(levelname)s %(message)s - -[formatter_console] -format=[%(levelname)s]: %(message)s \ No newline at end of file diff --git a/imap2thehive/imap2thehive.whitelists b/imap2thehive/imap2thehive.whitelists deleted file mode 100644 index 094c127..0000000 --- a/imap2thehive/imap2thehive.whitelists +++ /dev/null @@ -1,6 +0,0 @@ -# Exclude private IP addresses -172\.16\.\d{1,3}\.\d{1,3} -192\.168\.\d{1,3}\.\d{1,3} - -# Exclude my domains -\.rootshell\.be diff --git a/pipeline.yml b/pipeline.yml new file mode 100644 index 0000000..ad8c359 --- /dev/null +++ b/pipeline.yml @@ -0,0 +1,4 @@ +version: "2.0.0" +params: + platform: + name: "docker" diff --git a/repository.json b/repository.json new file mode 100644 index 0000000..4e5753f --- /dev/null +++ b/repository.json @@ -0,0 +1,8 @@ +{ + "name": "imap2thehive", + "description": "CronJob to sync our AlienVault email box with TheHive", + "owner": "Polaris", + "type": "tool", + "status": "unreleased", + "region-type": null +} diff --git a/imap2thehive/requirements.txt b/requirements.txt similarity index 91% rename from imap2thehive/requirements.txt rename to requirements.txt index efcf102..122d309 100644 --- a/imap2thehive/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ requests uuid python-magic thehive4py +chardet diff --git a/torwitness/Dockerfile b/torwitness/Dockerfile deleted file mode 100644 index 85e4c73..0000000 --- a/torwitness/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM phusion/baseimage -MAINTAINER Xavier Mertens - -RUN apt-get update && \ - apt-get install -y git wget sudo iptables python3-pip && \ - rm -rf /var/lib/apt/lists/* - -WORKDIR /opt - -# Install xlsxtract -RUN pip3 install openpyxl -RUN wget https://raw.githubusercontent.com/xme/toolbox/master/xlsxtract.py && \ - chmod a+x /opt/xlsxtract.py - -RUN git clone https://github.com/ChrisTruncer/EyeWitness.git - -WORKDIR /opt/EyeWitness - -RUN cd setup && \ - ./setup.sh - -WORKDIR /opt -RUN git clone https://github.com/susmithHCK/torghost.git -WORKDIR /opt/torghost -RUN chmod 0755 install.sh && \ - ./install.sh - -COPY entrypoint.sh /entrypoint.sh -RUN chmod 0755 /entrypoint.sh - -VOLUME /data - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/torwitness/README.md b/torwitness/README.md deleted file mode 100644 index 940dbc0..0000000 --- a/torwitness/README.md +++ /dev/null @@ -1,69 +0,0 @@ -Introduction -============ -Docker file to run Tor+EyeWitness. EyeWitness (https://github.com/ChrisTruncer/EyeWitness) is a tool to collect screenshots of websites. -The idea of the docker came after reading Micah Hoffman's blog post to browse new Onion website (https://webbreacher.com/2017/09/02/dark-web-report-torghost-eyewitness-goodness/). The docker has been created to extract Onion URL's from daily XLS files provided by @hunchy (https://twitter.com/hunchly/) - -Build -===== -$ docker built -t torwitness/latest Dockerfile . - -Usage -===== -$ docker run \ - --run \ - -it \ - -v /local_path:/data \ - --cap-add=NET_ADMIN --cap-add=NET_RAW \ - --env EW_TIMEOUT=30 \ - --env EW_MAX_RETRIES=5 \ - torwitness \ - urls.txt - -'urls.txt' is the text file with all the URLs that you want to visit through Tor/EyeWitness. This file must be available in the docker volume (/local_path/urls.txt). If you don't provide this file, the docker will search for .xlsx files in /data and extract URLs via my xlsxtract.py tool (https://github.com/xme/toolbox/blob/master/xlsxtract.py). The xlsx are expected to be the one provided by @hunchy. The results are stored in /data/results-YYYYMMDDHHMMSS/ - -WARNING: This Docker cannot be considered as safe because it has full access to the host network. This is required by torghost to properly intercept the HTTP traffic. - -Example -======= -$ cat $HOME/torwitness/urls.txt -https://blog.rootshell.be -https://isc.sans.edu -$ docker run --rm -it -v $HOME/torwitness:/data --cap-add=NET_ADMIN --cap-add=NET_RAW torwitness urls.txt -``` - _____ ____ _ _ - |_ _|__ _ __ / ___| |__ ___ ___| |_ - | |/ _ \| '__| | _| '_ \ / _ \/ __| __| - | | (_) | | | |_| | | | | (_) \__ \ |_ - |_|\___/|_| \____|_| |_|\___/|___/\__| - v2.0 - SusmithHCK | www.khromozome.com - - -[done] -[12:19:18] Configuring DNS resolv.conf file.. [done] - * Starting tor daemon... [ OK ] -[12:19:18] Starting tor service.. [done] -[12:19:19] setting up iptables rules [done] -[12:19:19] Fetching current IP... -[12:19:19] CURRENT IP : 51.15.79.107 -Using environment variables: -TIMEOUT=30 -MAX_RETRIES=3 -Found Onion URLs to process: -https://blog.rootshell.be -https://isc.sans.edu -################################################################################ -# EyeWitness # -################################################################################ - -Starting Web Requests (2 Hosts) -Attempting to screenshot https://blog.rootshell.be -Attempting to screenshot https://isc.sans.edu -Attempting to screenshot https://blog.rootshell.be -Attempting to screenshot https://isc.sans.edu -Finished in 22.2635469437 seconds -$ cd $HOME/torwitness -$ ls -results-20171025113745 urls.txt -$ cd results-20171025113745 -$ firefox result.html -``` diff --git a/torwitness/entrypoint.sh b/torwitness/entrypoint.sh deleted file mode 100644 index 3a7c1ab..0000000 --- a/torwitness/entrypoint.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -set -e -shopt -s nocaseglob - -# Start TorGhost -cd /opt/torghost -python torghost start - -test -z ${EW_TIMEOUT+w} && EW_TIMEOUT=30 -test -z ${EW_MAX_RETRIES+w} && EW_MAX_RETRIES=3 -echo "Using environment variables:" -echo "TIMEOUT=$EW_TIMEOUT" -echo "MAX_RETRIES=$EW_MAX_RETRIES" - -# Do we have an argument passed via the command line? -if [ -z ${1+x} ]; then - URLFILE=urls.txt - echo "No URL file defined, search for XLS files" - ls /data/*.xlsx >/dev/null - if [ "$?" != "0" ]; then - echo "No XLSX input file to process" - exit 1 - else - /opt/xlsxtract.py -w 'New Today' -c A -r 2- -s /data/*.xlsx >/data/$URLFILE - fi -else - URLFILE=`basename "$1"` - if [ ! -r "/data/$URLFILE" ]; then - echo "Cannot read URLS from $URLFILE" - exit 1 - fi -fi - -echo "Found Onion URLs to process:" -cat "/data/$URLFILE" - -# Remove previous results if exist -test -d /data/resuts && rm -r /data/results - -cd /opt/EyeWitness -python EyeWitness.py -d /data/results -f "/data/$URLFILE" --headless --no-prompt --timeout $EW_TIMEOUT --max-retries $EW_MAX_RETRIES -mv /data/results /data/results-`date +"%Y%m%d%H%M%S"` diff --git a/vipermonkey/Dockerfile b/vipermonkey/Dockerfile deleted file mode 100644 index bb8cdb5..0000000 --- a/vipermonkey/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM ubuntu:latest -MAINTAINER Xavier Mertens - -RUN apt-get update && apt-get -y upgrade -RUN apt-get -y install wget unzip python-pip - -WORKDIR /opt -RUN wget https://github.com/decalage2/ViperMonkey/archive/master.zip \ - && unzip master.zip - -WORKDIR /opt/ViperMonkey-master -RUN pip install -U -r requirements.txt -RUN python setup.py install - -WORKDIR /malware -ENTRYPOINT [ "/opt/ViperMonkey-master/vipermonkey/vmonkey.py" ] -CMD [ "-h" ] \ No newline at end of file diff --git a/vipermonkey/README.md b/vipermonkey/README.md deleted file mode 100644 index d5ef1c7..0000000 --- a/vipermonkey/README.md +++ /dev/null @@ -1,11 +0,0 @@ -Introduction -============ -Docker file to run vipermonkey (https://github.com/decalage2/ViperMonkey) from a Docker - -Build -===== -$ docker built -t vipermonkey/latest Dockerfile . - -Usage -===== -$ docker run --run -v /local/path:/malware vipermonkey/latest [options] ... \ No newline at end of file