diff --git a/Dockerfile b/Dockerfile index a1d754c..0407244 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,24 @@ -FROM python:3.10-bullseye@sha256:02c7cb92b8f23908de6457f7800c93b84ed8c6e7201da7935443d4c5eca7b381 +FROM python:3.12-bookworm@sha256:9bed8554e926c07c6f908841d5ee88c33e8df9236b191526bbce81a9062ab43a # Update and package installation RUN apt-get update && \ apt-get clean && \ apt-get install -y ca-certificates-java --no-install-recommends && \ - apt-get install -y openjdk-11-jdk p11-kit wkhtmltopdf libqt5gui5 && \ + apt-get install -y openjdk-17-jdk p11-kit wkhtmltopdf libqt5gui5 wget unzip && \ apt-get clean && \ update-ca-certificates -f # Get JADX Tool -ENV JADX_VERSION 1.4.7 +ENV JADX_VERSION 1.5.5 RUN \ - wget "https://github.com/skylot/jadx/releases/download/v$JADX_VERSION/jadx-$JADX_VERSION.zip" && \ - unzip "jadx-$JADX_VERSION.zip" + wget -q "https://github.com/skylot/jadx/releases/download/v$JADX_VERSION/jadx-$JADX_VERSION.zip" -O /tmp/jadx.zip && \ + mkdir -p /opt/jadx && \ + unzip -q /tmp/jadx.zip -d /opt/jadx && \ + chmod 0755 /opt/jadx/bin/jadx /opt/jadx/bin/jadx-gui && \ + rm /tmp/jadx.zip + +ENV PATH="/opt/jadx/bin:${PATH}" # Create user ARG uid=1000 @@ -22,7 +27,10 @@ ARG user=app ARG group=app RUN groupadd -g ${gid} ${group} \ - && useradd -u ${uid} -g ${group} -s /bin/sh ${user} + && useradd --create-home --home-dir /home/${user} \ + -u ${uid} -g ${group} -s /bin/sh ${user} + +ENV HOME=/home/${user} # Copy entrypoints COPY entrypoint/web_entrypoint.sh \ diff --git a/app/analysis.py b/app/analysis.py index bff2510..374ffdb 100755 --- a/app/analysis.py +++ b/app/analysis.py @@ -1,9 +1,6 @@ -from androguard.core.bytecodes.apk import APK -from androguard.core.bytecodes.dvm import DalvikVMFormat -from androguard.core.analysis.analysis import Analysis -from androguard.core.androconf import show_logging +from androguard.core.apk import APK from django.conf import settings -import logging, os, threading, hashlib, re, linecache, base64, urllib +import logging, os, threading, hashlib, re, linecache, base64, urllib, shutil, signal, subprocess, tempfile from app.models import * from pygments import highlight from pygments.lexers import PythonLexer @@ -45,6 +42,7 @@ def analyze_apk(task, scan_id): # Start the APK analysis global APK_PATH global DECOMPILE_PATH + scan = None try: scan = Scan.objects.get(pk=scan_id) APK_PATH = settings.BASE_DIR + scan.apk.url @@ -89,7 +87,7 @@ def analyze_apk(task, scan_id): task.update_state(state = 'STARTED', meta = {'current': scan.progress, 'total': 100, 'status': scan.status}) logger.debug(scan.status) - decompile_jadx() + decompile_jadx(APK_PATH, DECOMPILE_PATH) if (a.get_app_icon()): update_icon(scan, DECOMPILE_PATH + '/resources/' + a.get_app_icon()) scan.status = 'Finding vulnerabilities' @@ -107,19 +105,164 @@ def analyze_apk(task, scan_id): meta = {'current': scan.progress, 'total': 100, 'status': scan.status}) logger.debug(scan.status) except Exception as e: - scan.progress = 100 - scan.status = "Error" - scan.finished_on = datetime.now() - scan.save() - task.update_state(state = 'STARTED', - meta = {'current': scan.progress, 'total': 100, 'status': scan.status}) - logger.error(e) + if scan is not None: + scan.progress = 100 + scan.status = "Error" + scan.finished_on = datetime.now() + scan.save() + logger.exception(e) + raise + + +def _has_decompiled_output(output_path): + for directory in ('sources', 'resources'): + directory_path = os.path.join(output_path, directory) + if not os.path.isdir(directory_path): + continue + for _, _, files in os.walk(directory_path): + if files: + return True + return False + + +def _cleanup_decompile_output(output_path, marker_path): + shutil.rmtree(output_path, ignore_errors=True) + try: + os.remove(marker_path) + except FileNotFoundError: + pass + + +def _has_valid_completion_marker(marker_path): + try: + with open(marker_path, 'rb') as marker: + return marker.read() == b'complete\n' + except OSError: + return False + + +def _write_completion_marker(marker_path): + marker_directory = os.path.dirname(marker_path) or '.' + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode='w', + encoding='utf-8', + dir=marker_directory, + prefix=os.path.basename(marker_path) + '.', + delete=False, + ) as marker: + temporary_path = marker.name + marker.write('complete\n') + marker.flush() + os.fsync(marker.fileno()) + os.replace(temporary_path, marker_path) + except Exception: + if temporary_path is not None: + try: + os.remove(temporary_path) + except FileNotFoundError: + pass + raise + + +def _capture_output_tail(stream, tail, limit=4000): + for chunk in iter(lambda: stream.read(4096), b''): + tail.extend(chunk) + if len(tail) > limit: + del tail[:-limit] + + +def _terminate_process_group(process): + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + return + + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + +def decompile_jadx(apk_path=None, output_path=None, timeout=None): + apk_path = apk_path or APK_PATH + output_path = output_path or DECOMPILE_PATH + timeout = timeout or getattr(settings, 'JADX_TIMEOUT_SECONDS', 900) + marker_path = output_path + '.jadx-complete' + + if _has_valid_completion_marker(marker_path) and _has_decompiled_output(output_path): + return + + _cleanup_decompile_output(output_path, marker_path) + command = ['jadx', '-d', output_path, apk_path] + output_tail = bytearray() + try: + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + output_reader = threading.Thread( + target=_capture_output_tail, + args=(process.stdout, output_tail), + daemon=True, + ) + output_reader.start() + process.wait(timeout=timeout) + except subprocess.TimeoutExpired as exc: + _terminate_process_group(process) + output_reader.join(timeout=5) + _cleanup_decompile_output(output_path, marker_path) + error_output = bytes(output_tail[-4000:]).decode( + 'utf-8', + errors='replace', + ).strip() + detail = ': {}'.format(error_output) if error_output else '' + raise RuntimeError( + 'JADX decompilation timed out after {} seconds{}'.format( + timeout, + detail, + ) + ) from exc + except OSError as exc: + if 'process' in locals(): + _terminate_process_group(process) + _cleanup_decompile_output(output_path, marker_path) + raise RuntimeError('Unable to execute JADX: {}'.format(exc)) from exc + + output_reader.join(timeout=5) + error_output = bytes(output_tail[-4000:]).decode( + 'utf-8', + errors='replace', + ).strip() + if process.returncode not in (0, 3): + _cleanup_decompile_output(output_path, marker_path) + raise RuntimeError( + 'JADX decompilation failed with exit code {}: {}'.format( + process.returncode, + error_output or 'No error output', + ) + ) + + if not _has_decompiled_output(output_path): + _cleanup_decompile_output(output_path, marker_path) + raise RuntimeError( + 'JADX decompilation produced no source or resource files' + ) + + if process.returncode == 3: + logger.warning( + 'JADX completed with partial decompilation (exit code 3); ' + 'continuing with usable output. Output tail: %s', + error_output or 'No error output', + ) -def decompile_jadx(): - if (not os.path.isdir(DECOMPILE_PATH)): - #execute jadx command - os.system('jadx -d {} {}'.format(DECOMPILE_PATH, APK_PATH)) - # now we have sources/resources decompiled + _write_completion_marker(marker_path) def update_icon(scan, path): encoded_string = '' @@ -432,4 +575,4 @@ def get_info_database(scan, path): f = open(path) table_info = f.read() db = DatabaseInfo(scan=scan, table='None', info=table_info) - db.save() \ No newline at end of file + db.save() diff --git a/app/tests.py b/app/tests.py index 7ce503c..37a537c 100755 --- a/app/tests.py +++ b/app/tests.py @@ -1,3 +1,277 @@ -from django.test import TestCase +import os +import json +from io import BytesIO +import signal +import subprocess +import tempfile +from unittest.mock import MagicMock, call, patch -# Create your tests here. +from django.test import SimpleTestCase, override_settings + +from app import analysis +from app.worker import tasks + + +class DecompileJadxTests(SimpleTestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.output_path = os.path.join(self.temp_dir.name, 'decompiled') + self.apk_path = os.path.join(self.temp_dir.name, 'sample.apk') + + def tearDown(self): + self.temp_dir.cleanup() + + def create_valid_output(self): + sources_path = os.path.join(self.output_path, 'sources') + os.makedirs(sources_path) + with open(os.path.join(sources_path, 'Main.java'), 'w') as source: + source.write('class Main {}') + + @patch('app.analysis.subprocess.Popen') + def test_decompile_jadx_runs_in_new_session_and_marks_valid_output(self, popen): + process = MagicMock(pid=123, returncode=0) + process.stdout = BytesIO() + process.wait.side_effect = lambda timeout: self.create_valid_output() + popen.return_value = process + + analysis.decompile_jadx(self.apk_path, self.output_path, timeout=30) + + popen.assert_called_once_with( + ['jadx', '-d', self.output_path, self.apk_path], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + process.wait.assert_called_once_with(timeout=30) + self.assertTrue(os.path.isfile(self.output_path + '.jadx-complete')) + + @patch('app.analysis.subprocess.Popen') + def test_decompile_jadx_cleans_stale_output_before_retry(self, popen): + os.makedirs(os.path.join(self.output_path, 'sources')) + stale_path = os.path.join(self.output_path, 'sources', 'stale.java') + with open(stale_path, 'w') as stale_file: + stale_file.write('stale') + process = MagicMock(pid=123, returncode=0) + process.stdout = BytesIO() + process.wait.side_effect = lambda timeout: self.create_valid_output() + popen.return_value = process + + analysis.decompile_jadx(self.apk_path, self.output_path, timeout=30) + + self.assertFalse(os.path.exists(stale_path)) + self.assertTrue(os.path.isfile(self.output_path + '.jadx-complete')) + + @patch('app.analysis.subprocess.Popen') + def test_decompile_jadx_retries_when_completion_marker_is_invalid(self, popen): + self.create_valid_output() + marker_path = self.output_path + '.jadx-complete' + with open(marker_path, 'w') as marker: + marker.write('') + process = MagicMock(pid=123, returncode=0) + process.stdout = BytesIO() + process.wait.side_effect = lambda timeout: self.create_valid_output() + popen.return_value = process + + analysis.decompile_jadx(self.apk_path, self.output_path, timeout=30) + + popen.assert_called_once() + with open(marker_path, 'rb') as marker: + self.assertEqual(marker.read(), b'complete\n') + + @patch('app.analysis.os.replace', wraps=os.replace) + @patch('app.analysis.subprocess.Popen') + def test_decompile_jadx_writes_completion_marker_atomically( + self, + popen, + replace, + ): + process = MagicMock(pid=123, returncode=0) + process.stdout = BytesIO() + process.wait.side_effect = lambda timeout: self.create_valid_output() + popen.return_value = process + + analysis.decompile_jadx(self.apk_path, self.output_path, timeout=30) + + source_path, destination_path = replace.call_args.args + self.assertEqual(destination_path, self.output_path + '.jadx-complete') + self.assertEqual( + os.path.dirname(source_path), + os.path.dirname(destination_path), + ) + self.assertNotEqual(source_path, destination_path) + + @patch('app.analysis.subprocess.Popen') + def test_decompile_jadx_raises_on_non_zero_exit(self, popen): + process = MagicMock(pid=123, returncode=1) + process.stdout = BytesIO(b'res1 must be zero!') + process.wait.side_effect = lambda timeout: os.mkdir(self.output_path) + popen.return_value = process + + with self.assertRaisesRegex(RuntimeError, 'res1 must be zero'): + analysis.decompile_jadx(self.apk_path, self.output_path, timeout=30) + + self.assertFalse(os.path.exists(self.output_path)) + + @patch('app.analysis.subprocess.Popen') + def test_decompile_jadx_limits_failure_output_to_last_4000_bytes(self, popen): + process = MagicMock(pid=123, returncode=1) + process.stdout = BytesIO(b'prefix' + (b'x' * 4000)) + process.wait.side_effect = lambda timeout: os.mkdir(self.output_path) + popen.return_value = process + + with self.assertRaises(RuntimeError) as raised: + analysis.decompile_jadx(self.apk_path, self.output_path, timeout=30) + + message = str(raised.exception) + self.assertNotIn('prefix', message) + self.assertTrue(message.endswith('x' * 4000)) + + @patch('app.analysis.logger.warning') + @patch('app.analysis.subprocess.Popen') + def test_decompile_jadx_accepts_partial_exit_with_valid_output( + self, + popen, + warning, + ): + process = MagicMock(pid=123, returncode=3) + process.stdout = BytesIO(b'ERROR - finished with errors, count: 12') + process.wait.side_effect = lambda timeout: self.create_valid_output() + popen.return_value = process + + analysis.decompile_jadx(self.apk_path, self.output_path, timeout=30) + + self.assertTrue(os.path.isfile(self.output_path + '.jadx-complete')) + warning.assert_called_once_with( + 'JADX completed with partial decompilation (exit code 3); ' + 'continuing with usable output. Output tail: %s', + 'ERROR - finished with errors, count: 12', + ) + + @patch('app.analysis.subprocess.Popen') + def test_decompile_jadx_rejects_partial_exit_with_empty_output(self, popen): + process = MagicMock(pid=123, returncode=3) + process.stdout = BytesIO(b'ERROR - finished with errors, count: 12') + process.wait.side_effect = lambda timeout: os.mkdir(self.output_path) + popen.return_value = process + + with self.assertRaisesRegex( + RuntimeError, + 'produced no source or resource files', + ): + analysis.decompile_jadx(self.apk_path, self.output_path, timeout=30) + + self.assertFalse(os.path.exists(self.output_path)) + self.assertFalse(os.path.exists(self.output_path + '.jadx-complete')) + + @patch('app.analysis.os.killpg') + @patch('app.analysis.subprocess.Popen') + def test_decompile_jadx_terminates_process_group_on_timeout(self, popen, killpg): + process = MagicMock(pid=123) + process.stdout = BytesIO() + process.wait.side_effect = [ + subprocess.TimeoutExpired(cmd='jadx', timeout=30), + 0, + ] + popen.return_value = process + + with self.assertRaisesRegex(RuntimeError, 'timed out after 30 seconds'): + analysis.decompile_jadx(self.apk_path, self.output_path, timeout=30) + + killpg.assert_called_once_with(123, signal.SIGTERM) + self.assertEqual( + process.wait.call_args_list, + [call(timeout=30), call(timeout=5)], + ) + + @patch('app.analysis.subprocess.Popen') + def test_decompile_jadx_rejects_empty_success_output(self, popen): + process = MagicMock(pid=123, returncode=0) + process.stdout = BytesIO() + process.wait.side_effect = lambda timeout: os.mkdir(self.output_path) + popen.return_value = process + + with self.assertRaisesRegex(RuntimeError, 'produced no source or resource files'): + analysis.decompile_jadx(self.apk_path, self.output_path, timeout=30) + + self.assertFalse(os.path.exists(self.output_path)) + self.assertFalse(os.path.exists(self.output_path + '.jadx-complete')) + + @override_settings(BASE_DIR='/tmp') + @patch('app.analysis.get_info_certificate') + @patch('app.analysis.get_info_apk') + @patch('app.analysis.set_hash_app') + @patch('app.analysis.APK') + @patch('app.analysis.Scan.objects.get') + @patch('app.analysis.decompile_jadx') + def test_decompile_error_sets_terminal_scan_status( + self, + decompile_jadx, + get_scan, + apk, + set_hash_app, + get_info_apk, + get_info_certificate, + ): + scan = MagicMock() + scan.apk.url = '/sample.apk' + get_scan.return_value = scan + set_hash_app.return_value = scan + get_info_apk.return_value = scan + decompile_jadx.side_effect = RuntimeError('decompilation failed') + task = MagicMock() + + with self.assertRaisesRegex(RuntimeError, 'decompilation failed'): + analysis.analyze_apk(task, 1) + + self.assertEqual(scan.status, 'Error') + self.assertEqual(scan.progress, 100) + self.assertIsNotNone(scan.finished_on) + scan.save.assert_called() + self.assertFalse( + any( + update.kwargs.get('meta', {}).get('status') == 'Error' + for update in task.update_state.call_args_list + ) + ) + + +class ScanStateTests(SimpleTestCase): + @patch('app.worker.tasks.AsyncResult') + @patch('app.worker.tasks.Scan.objects.get') + def test_scan_state_serializes_celery_failure_exception( + self, + get_scan, + async_result, + ): + scan = MagicMock(task='task-id', progress=100, status='Error') + get_scan.return_value = scan + async_result.return_value.info = RuntimeError('decompilation failed') + + response = tasks.scan_state(MagicMock(), 1) + + self.assertEqual(response.status_code, 200) + self.assertEqual( + json.loads(response.content), + { + 'current': 100, + 'total': 100, + 'status': 'Error', + 'error': 'decompilation failed', + }, + ) + + @patch('app.worker.tasks.AsyncResult') + @patch('app.worker.tasks.Scan.objects.get') + def test_scan_state_returns_ordinary_result_dict_unchanged( + self, + get_scan, + async_result, + ): + progress = {'current': 40, 'total': 100, 'status': 'In Progress'} + get_scan.return_value = MagicMock(task='task-id') + async_result.return_value.info = None + async_result.return_value.result = progress + + response = tasks.scan_state(MagicMock(), 1) + + self.assertEqual(json.loads(response.content), progress) diff --git a/app/worker/tasks.py b/app/worker/tasks.py index 74d438c..bcff433 100644 --- a/app/worker/tasks.py +++ b/app/worker/tasks.py @@ -23,4 +23,14 @@ def scan_state(request, id): data = job.result except Exception as e: logger.error(e) - return HttpResponse(json.dumps(data), content_type='application/json') \ No newline at end of file + data = e + + if isinstance(data, BaseException): + data = { + 'current': scan.progress, + 'total': 100, + 'status': scan.status, + 'error': str(data), + } + + return HttpResponse(json.dumps(data, default=str), content_type='application/json') diff --git a/docker-compose.prod.yaml b/docker-compose.prod.yaml index cb9bf5a..efe5727 100644 --- a/docker-compose.prod.yaml +++ b/docker-compose.prod.yaml @@ -1,7 +1,7 @@ version: '3.8' services: db: - image: postgres:16-bullseye@sha256:7174d2a352ad138906e3dc4a28a5d11b5a158180e9bff80beed8c8cc346f874c + image: postgres:16-bookworm@sha256:92620daddcd947f8d5ab5ba66e848702fe443d87fed30c4cea8e389fd78dfc55 env_file: - ./.env.example environment: @@ -28,7 +28,7 @@ services: - db restart: on-failure nginx: - image: nginx:stable-bullseye@sha256:ee187e563496b690edaab157f89db924cd35fab42631309f4d62957baecf7d6c + image: nginx:stable-bookworm@sha256:552e7481ca93ffccd046aa658dbbed22caefbc09c66fa7cd247cbb90b8a5c609 ports: - "443:443" volumes: @@ -40,7 +40,7 @@ services: - web restart: on-failure rabbitmq: - image: rabbitmq:3.13.0-management@sha256:ba406c7daaef53b59c92a13db37a27d0425579a40c59266d10022a40a8ba7242 + image: rabbitmq:4.2-management@sha256:a570a0204bbbd357cd7fdac90f9f1b72f644d06dda4c49799a9452e5b9941dfc env_file: - ./.env.example environment: diff --git a/docker-compose.yaml b/docker-compose.yaml index 7d49fcb..36cd608 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,7 +1,7 @@ version: '3.8' services: db: - image: postgres:16-bullseye@sha256:7174d2a352ad138906e3dc4a28a5d11b5a158180e9bff80beed8c8cc346f874c + image: postgres:16-bookworm@sha256:92620daddcd947f8d5ab5ba66e848702fe443d87fed30c4cea8e389fd78dfc55 env_file: - ./.env.example environment: @@ -28,7 +28,7 @@ services: - db restart: on-failure nginx: - image: nginx:stable-bullseye@sha256:ee187e563496b690edaab157f89db924cd35fab42631309f4d62957baecf7d6c + image: nginx:stable-bookworm@sha256:552e7481ca93ffccd046aa658dbbed22caefbc09c66fa7cd247cbb90b8a5c609 ports: - "8888:8888" volumes: @@ -39,7 +39,7 @@ services: - web restart: on-failure rabbitmq: - image: rabbitmq:3.13.0-management@sha256:ba406c7daaef53b59c92a13db37a27d0425579a40c59266d10022a40a8ba7242 + image: rabbitmq:4.2-management@sha256:a570a0204bbbd357cd7fdac90f9f1b72f644d06dda4c49799a9452e5b9941dfc env_file: - ./.env.example environment: diff --git a/requirements.txt b/requirements.txt index 2474e1f..468db9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ -amqp==5.2.0 -androguard==3.4.0a1 +amqp==5.3.1 +androguard==4.1.4 asgiref==3.7.2 asn1crypto==1.5.1 asttokens==2.4.1 beautifulsoup4==4.12.3 -billiard==3.6.4.0 -celery==5.2.2 +billiard==4.2.2 +celery==5.6.3 certifi==2024.7.4 charset-normalizer==3.3.2 click==8.1.7 @@ -18,7 +18,7 @@ coreapi==2.3.3 coreschema==0.0.4 cycler==0.12.1 decorator==5.1.1 -Django==4.2.28 +Django==4.2.30 django-bootstrap4==3.0.1 django-extensions==3.1.3 django-filter==2.4.0 @@ -26,18 +26,18 @@ django-fontawesome-5==1.0.18 django-getenv==1.3.2 django-widget-tweaks==1.4.8 djangorestframework==3.15.2 -drf-yasg==1.20.0 +drf-yasg==1.21.15 executing==2.0.1 fonttools==4.61.0 -idna==3.7 +idna==3.15 inflection==0.5.1 ipython==8.22.2 itypes==1.2.0 jedi==0.19.1 Jinja2==3.1.6 kiwisolver==1.4.5 -kombu==5.3.5 -lxml==5.1.0 +kombu==5.6.2 +lxml==6.1.0 MarkupSafe==2.1.5 matplotlib==3.8.3 matplotlib-inline==0.1.6 @@ -47,18 +47,18 @@ packaging==23.2 parso==0.8.3 pdfkit==0.6.1 pexpect==4.9.0 -pillow==12.1.1 +pillow==12.2.0 prompt-toolkit==3.0.43 psycopg2-binary==2.9.9 psycopg2==2.9.9 ptyprocess==0.7.0 pure-eval==0.2.2 pydot==2.0.0 -Pygments==2.15.0 +Pygments==2.20.0 pyparsing==3.1.2 python-dateutil==2.9.0.post0 pytz==2024.1 -requests==2.32.4 +requests==2.33.0 ruamel.yaml==0.18.6 ruamel.yaml.clib==0.2.8 six==1.16.0 @@ -67,9 +67,10 @@ SQLAlchemy==1.4.23 sqlparse==0.5.4 stack-data==0.6.3 traitlets==5.14.1 +tzlocal==5.4.4 uritemplate==4.1.1 -urllib3==2.6.3 -uWSGI==2.0.22 +urllib3==2.7.0 +uWSGI==2.0.31 vine==5.1.0 wcwidth==0.2.13 zipp>=3.19.1