diff --git a/.github/workflows/android-tests.yml b/.github/workflows/android-tests.yml new file mode 100644 index 0000000..495b522 --- /dev/null +++ b/.github/workflows/android-tests.yml @@ -0,0 +1,41 @@ +name: Android Tests + +on: + push: + branches: + - master + pull_request: + +permissions: + contents: read + +jobs: + unit-tests: + name: Unit tests + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + + - name: Set up Gradle + uses: gradle/actions/setup-gradle@v4 + with: + gradle-version: "9.5.0" + + - name: Run unit tests + working-directory: android_app + run: gradle test + + - name: Upload test reports + if: failure() + uses: actions/upload-artifact@v4 + with: + name: test-reports + path: android_app/app/build/reports/tests/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..b900fe0 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,128 @@ +name: Cut Release + +# Server-side equivalent of scripts/release.sh --push: bump the version, fold the +# changelog, tag, build the signed APK, and publish the GitHub Release — all from +# the Actions "Run workflow" button (or an API dispatch), no local checkout needed. +# +# The tag is pushed with the workflow token, which deliberately does NOT trigger +# the tag-driven android-release.yml (GitHub suppresses workflow-to-workflow +# triggers); this workflow builds and publishes the release itself instead, so +# the two paths never double-release. Local scripts/release.sh --push keeps +# working unchanged via android-release.yml. + +on: + workflow_dispatch: + inputs: + bump: + description: "Version part to bump" + required: true + type: choice + default: patch + options: + - patch + - minor + - major + +permissions: + contents: write + +concurrency: + group: release + cancel-in-progress: false + +jobs: + release: + name: Bump, tag, build, publish + runs-on: ubuntu-latest + + steps: + - name: Ensure the workflow runs from master + if: github.ref != 'refs/heads/master' + run: | + echo "Releases must be cut from master (got ${GITHUB_REF})." >&2 + exit 1 + + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 # release.sh checks existing tags + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + + - name: Set up Gradle + uses: gradle/actions/setup-gradle@v4 + with: + gradle-version: "9.5.0" + + - name: Bump version, fold changelog, tag + id: bump + shell: bash + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + scripts/release.sh "${{ inputs.bump }}" --skip-build + tag="$(git describe --tags --exact-match HEAD)" + echo "tag=${tag}" >> "${GITHUB_OUTPUT}" + # The commit body carries the changelog bullets release.sh extracted. + git log -1 --format=%b > release-notes.md + + - name: Restore release keystore + shell: bash + run: | + mkdir -p android_app/release + printf '%s' "${ANDROID_RELEASE_KEYSTORE_BASE64}" | base64 --decode > android_app/release/expense-notification-release.jks + { + printf 'RELEASE_STORE_FILE=release/expense-notification-release.jks\n' + printf 'RELEASE_STORE_PASSWORD=%s\n' "${ANDROID_RELEASE_STORE_PASSWORD}" + printf 'RELEASE_KEY_ALIAS=%s\n' "${ANDROID_RELEASE_KEY_ALIAS}" + printf 'RELEASE_KEY_PASSWORD=%s\n' "${ANDROID_RELEASE_KEY_PASSWORD}" + } > android_app/keystore.properties + env: + ANDROID_RELEASE_KEYSTORE_BASE64: ${{ secrets.ANDROID_RELEASE_KEYSTORE_BASE64 }} + ANDROID_RELEASE_STORE_PASSWORD: ${{ secrets.ANDROID_RELEASE_STORE_PASSWORD }} + ANDROID_RELEASE_KEY_ALIAS: ${{ secrets.ANDROID_RELEASE_KEY_ALIAS }} + ANDROID_RELEASE_KEY_PASSWORD: ${{ secrets.ANDROID_RELEASE_KEY_PASSWORD }} + + - name: Run unit tests + working-directory: android_app + run: gradle test + + - name: Build signed release APK + working-directory: android_app + run: gradle assembleRelease + + - name: Name release APK + id: apk + shell: bash + run: | + apk_name="ExpenseCapture-${{ steps.bump.outputs.tag }}.apk" + apk_path="android_app/app/build/outputs/apk/release/${apk_name}" + cp android_app/app/build/outputs/apk/release/app-release.apk "${apk_path}" + echo "name=${apk_name}" >> "${GITHUB_OUTPUT}" + echo "path=${apk_path}" >> "${GITHUB_OUTPUT}" + + # Only push the version-bump commit and tag once the build has succeeded, + # so a failed build leaves master untouched and the run fully retryable. + - name: Push release commit and tag + shell: bash + run: | + git push origin HEAD:master + git push origin "${{ steps.bump.outputs.tag }}" + + - name: Upload APK artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.apk.outputs.name }} + path: ${{ steps.apk.outputs.path }} + if-no-files-found: error + + - name: Publish GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.bump.outputs.tag }} + body_path: release-notes.md + files: ${{ steps.apk.outputs.path }} diff --git a/CHANGELOG.md b/CHANGELOG.md index c73d880..3e77c62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # Changelog ## Unreleased +- Fix deleting a bundled parser/output config not actually disabling it: the parser kept a hardcoded copy of every bundled config and fell back to it even when the config was hidden. Bundled assets are now the single source of defaults, so Delete/Restore in the config UI really controls what parses. +- Fix grouped-thousands amounts without decimals: `€1.234` (and `1,234`) now parse as 1234, not 1.23. +- Fix a crash when tapping Fill or Open Expense Manager with the output app not installed; the candidate is no longer marked processed for a fill that never happened. +- Make the `dropZeroAmount` global config switch actually control the zero-amount filter (it was previously always on). +- Fix the config editor corrupting regex patterns that contain a literal backslash-u sequence when a config was opened and saved. +- Ask for confirmation before "Clear local queue" deletes captured notifications. +- Dedupe captures with a SHA-256 body hash instead of a 32-bit hash, removing the (tiny) chance of two different bank SMS colliding into one key and silently dropping an expense. +- Performance: compile every parser regex once per config load instead of on each notification; re-parse stored candidates only when the parser config actually changes (and write the result back) instead of on every list read; run notification parsing/database writes and the review-queue load off the main thread; share one database connection instead of opening one per event. +- Restrict the form-filling accessibility service to the configured output app via the system-side package filter (it previously woke up for events from every app), and expire an abandoned fill after 15 minutes. +- Prune processed/skipped candidates older than 90 days so the queue and database stop growing forever; unreviewed candidates are kept indefinitely. +- Skip a parser rule whose regex does not compile instead of crashing the notification listener. +- Run the unit-test suite in CI on every push and pull request. +- Add a "Cut Release" GitHub Actions workflow: releases can now be triggered from the Actions tab (choose patch/minor/major); it bumps the version, folds the changelog, tags, runs the tests, builds the signed APK, and publishes the GitHub Release — the server-side equivalent of `scripts/release.sh --push`. ## v1.0.1 - 2026-06-30 - Fix bank SMS after the first being silently dropped: messaging apps (e.g. Textra) post every SMS from one sender under a single conversation notification, so every Bank of Cyprus SMS shared one notification key and collided on the queue's unique-key constraint after the first capture. The dedupe key now folds in the message body, so each distinct SMS is queued while re-scanning the same still-active notification still dedupes. Card-app notifications (Revolut, Google Wallet) keep a unique key per transaction, so identical charges still queue separately. diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/BaseActivity.java b/android_app/app/src/main/java/dev/fanis/expensenotification/BaseActivity.java index dcdcda9..e16297b 100644 --- a/android_app/app/src/main/java/dev/fanis/expensenotification/BaseActivity.java +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/BaseActivity.java @@ -37,13 +37,9 @@ abstract class BaseActivity extends Activity { /** Wraps a vertical content column in a scroller with status/navigation-bar insets applied. */ protected ScrollView scrollRoot(LinearLayout root) { Window window = getWindow(); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - window.setStatusBarColor(COLOR_TEAL); - window.setNavigationBarColor(Color.WHITE); - } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - window.getDecorView().setSystemUiVisibility(View.SYSTEM_UI_FLAG_LIGHT_NAVIGATION_BAR); - } + window.setStatusBarColor(COLOR_TEAL); + window.setNavigationBarColor(Color.WHITE); + window.getDecorView().setSystemUiVisibility(View.SYSTEM_UI_FLAG_LIGHT_NAVIGATION_BAR); ScrollView scroll = new ScrollView(this); scroll.setFillViewport(true); scroll.setBackgroundColor(COLOR_BG); @@ -112,27 +108,25 @@ protected ScrollView scrollRoot(LinearLayout root) { header.setPadding(dp(26), fallbackTop + dp(20), dp(26), dp(20)); root.setPadding(horizontalPadding, contentTopPadding, horizontalPadding, dp(16) + fallbackBottom); scroll.setClipToPadding(false); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT_WATCH) { - scroll.setOnApplyWindowInsetsListener((view, insets) -> { - int top = fallbackTop; - int bottom = fallbackBottom; - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { - Insets systemBars = insets.getInsets(WindowInsets.Type.systemBars()); - top = Math.max(top, systemBars.top); - bottom = Math.max(bottom, systemBars.bottom); - } else { - top = Math.max(top, insets.getSystemWindowInsetTop()); - bottom = Math.max(bottom, insets.getSystemWindowInsetBottom()); - } - header.setPadding(dp(26), top + dp(20), dp(26), dp(20)); - root.setPadding( - horizontalPadding, - contentTopPadding, - horizontalPadding, - dp(16) + bottom); - return insets; - }); - } + scroll.setOnApplyWindowInsetsListener((view, insets) -> { + int top = fallbackTop; + int bottom = fallbackBottom; + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + Insets systemBars = insets.getInsets(WindowInsets.Type.systemBars()); + top = Math.max(top, systemBars.top); + bottom = Math.max(bottom, systemBars.bottom); + } else { + top = Math.max(top, insets.getSystemWindowInsetTop()); + bottom = Math.max(bottom, insets.getSystemWindowInsetBottom()); + } + header.setPadding(dp(26), top + dp(20), dp(26), dp(20)); + root.setPadding( + horizontalPadding, + contentTopPadding, + horizontalPadding, + dp(16) + bottom); + return insets; + }); outer.addView(header, new LinearLayout.LayoutParams( LinearLayout.LayoutParams.MATCH_PARENT, LinearLayout.LayoutParams.WRAP_CONTENT)); @@ -192,10 +186,8 @@ protected Button button(String label) { button.setMinHeight(dp(46)); button.setPadding(dp(18), 0, dp(18), 0); button.setBackground(rounded(COLOR_TEAL, COLOR_TEAL, 5)); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - button.setElevation(0f); - button.setStateListAnimator(null); - } + button.setElevation(0f); + button.setStateListAnimator(null); LinearLayout.LayoutParams params = new LinearLayout.LayoutParams( LinearLayout.LayoutParams.MATCH_PARENT, LinearLayout.LayoutParams.WRAP_CONTENT); @@ -308,4 +300,32 @@ protected void requestIgnoreBatteryOptimizations() { protected static String emptyDash(String text) { return text == null || text.isEmpty() ? "-" : text; } + + protected boolean isNotificationListenerEnabled() { + String enabled = Settings.Secure.getString(getContentResolver(), "enabled_notification_listeners"); + return containsComponent(enabled, ExpenseNotificationListener.class.getName()); + } + + protected boolean isAccessibilityServiceEnabled() { + String enabled = Settings.Secure.getString(getContentResolver(), "enabled_accessibility_services"); + return "1".equals(Settings.Secure.getString(getContentResolver(), "accessibility_enabled")) && + containsComponent(enabled, ExpenseEntryAccessibilityService.class.getName()); + } + + // The Settings.Secure lists are ':'-separated flattened components; match each + // entry exactly (long and short form) rather than by substring, so another + // package whose name merely contains ours can't count as enabled. + private boolean containsComponent(String enabled, String className) { + if (enabled == null || enabled.isEmpty()) { + return false; + } + String flattenedComponent = getPackageName() + "/" + className; + String shortComponent = getPackageName() + "/" + className.replace(getPackageName() + ".", "."); + for (String component : enabled.split(":")) { + if (component.equals(flattenedComponent) || component.equals(shortComponent)) { + return true; + } + } + return false; + } } diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/Candidate.java b/android_app/app/src/main/java/dev/fanis/expensenotification/Candidate.java index e8226c9..83a2b84 100644 --- a/android_app/app/src/main/java/dev/fanis/expensenotification/Candidate.java +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/Candidate.java @@ -3,6 +3,12 @@ import java.math.BigDecimal; final class Candidate { + static final String STATUS_NEW = "NEW"; + static final String STATUS_SKIPPED = "SKIPPED"; + static final String STATUS_PROCESSED = "PROCESSED"; + static final String TYPE_EXPENSE = "EXPENSE"; + static final String TYPE_INCOME = "INCOME"; + long id; String notificationKey; String packageName; @@ -17,12 +23,12 @@ final class Candidate { String suggestedCategory; String suggestedPaymentMethod; String note = ""; - String transactionType = "EXPENSE"; + String transactionType = TYPE_EXPENSE; long postedAt; String status; boolean isIncome() { - return "INCOME".equals(transactionType); + return TYPE_INCOME.equals(transactionType); } boolean hasAmount() { diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/CandidateDb.java b/android_app/app/src/main/java/dev/fanis/expensenotification/CandidateDb.java index 0d289b1..178c289 100644 --- a/android_app/app/src/main/java/dev/fanis/expensenotification/CandidateDb.java +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/CandidateDb.java @@ -11,10 +11,32 @@ final class CandidateDb extends SQLiteOpenHelper { private static final String DB_NAME = "expense_candidates.db"; - private static final int DB_VERSION = 3; + private static final int DB_VERSION = 4; + // Settled (processed/skipped) candidates older than this are pruned so the + // database and the review screen don't grow forever. NEW candidates are kept + // indefinitely: they still need the user's decision. + static final long SETTLED_RETENTION_MS = 90L * 24 * 60 * 60 * 1000; + + private static CandidateDb instance; + private final Context appContext; - CandidateDb(Context context) { + static synchronized CandidateDb getInstance(Context context) { + if (instance == null) { + instance = new CandidateDb(context.getApplicationContext()); + } + return instance; + } + + /** Tests run against per-test data directories; lets them drop the cached helper. */ + static synchronized void resetInstanceForTesting() { + if (instance != null) { + instance.close(); + instance = null; + } + } + + private CandidateDb(Context context) { super(context, DB_NAME, null, DB_VERSION); this.appContext = context.getApplicationContext(); } @@ -39,7 +61,8 @@ public void onCreate(SQLiteDatabase db) { "transaction_type TEXT NOT NULL DEFAULT 'EXPENSE'," + "posted_at INTEGER NOT NULL," + "status TEXT NOT NULL DEFAULT 'NEW'," + - "created_at INTEGER NOT NULL)"); + "created_at INTEGER NOT NULL," + + "parsed_revision INTEGER NOT NULL DEFAULT -1)"); } @Override @@ -52,9 +75,14 @@ public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) { db.execSQL("ALTER TABLE candidates ADD COLUMN note TEXT NOT NULL DEFAULT ''"); db.execSQL("ALTER TABLE candidates ADD COLUMN transaction_type TEXT NOT NULL DEFAULT 'EXPENSE'"); } + if (oldVersion < 4) { + // -1 marks rows parsed by an unknown (pre-column) config; they reparse once. + db.execSQL("ALTER TABLE candidates ADD COLUMN parsed_revision INTEGER NOT NULL DEFAULT -1"); + } } long insertIfNew(Candidate candidate) { + pruneSettled(); ContentValues values = new ContentValues(); values.put("notification_key", candidate.notificationKey); values.put("package_name", candidate.packageName); @@ -69,19 +97,18 @@ long insertIfNew(Candidate candidate) { values.put("suggested_category", value(candidate.suggestedCategory)); values.put("suggested_payment_method", value(candidate.suggestedPaymentMethod)); values.put("note", value(candidate.note)); - values.put("transaction_type", value(candidate.transactionType, "EXPENSE")); + values.put("transaction_type", value(candidate.transactionType, Candidate.TYPE_EXPENSE)); values.put("posted_at", candidate.postedAt); - values.put("status", value(candidate.status, "NEW")); + values.put("status", value(candidate.status, Candidate.STATUS_NEW)); values.put("created_at", System.currentTimeMillis()); + values.put("parsed_revision", ConfigRevision.current(appContext)); return getWritableDatabase().insertWithOnConflict("candidates", null, values, SQLiteDatabase.CONFLICT_IGNORE); } - List listNew() { - return queryByStatus("NEW"); - } - List listAll() { ArrayList items = new ArrayList<>(); + ArrayList stale = new ArrayList<>(); + long currentRevision = ConfigRevision.current(appContext); try (Cursor cursor = getReadableDatabase().query( "candidates", null, @@ -90,10 +117,19 @@ List listAll() { null, null, "posted_at DESC, id DESC")) { + int revisionIndex = cursor.getColumnIndex("parsed_revision"); while (cursor.moveToNext()) { - items.add(fromCursor(cursor)); + Candidate candidate = fromCursor(cursor); + items.add(candidate); + long parsedRevision = revisionIndex < 0 ? -1 : cursor.getLong(revisionIndex); + if (parsedRevision != currentRevision) { + stale.add(candidate); + } } } + if (!stale.isEmpty()) { + reparseStale(stale, currentRevision); + } return items; } @@ -107,21 +143,62 @@ int deleteAll() { return getWritableDatabase().delete("candidates", null, null); } - private List queryByStatus(String status) { - ArrayList items = new ArrayList<>(); - try (Cursor cursor = getReadableDatabase().query( + int pruneSettled() { + long cutoff = System.currentTimeMillis() - SETTLED_RETENTION_MS; + return getWritableDatabase().delete( "candidates", - null, - "status = ?", - new String[]{status}, - null, - null, - "posted_at DESC, id DESC")) { - while (cursor.moveToNext()) { - items.add(fromCursor(cursor)); + "status != ? AND posted_at < ?", + new String[]{Candidate.STATUS_NEW, String.valueOf(cutoff)}); + } + + // Re-derives parsed fields (payee, payment method, note, category, amount, + // transaction type) from the stored SMS so existing candidates reflect the + // current parser config. Identity (id, notification key, raw title/body) and + // workflow status are preserved. Runs only for rows whose parsed_revision is + // behind the current config revision, and writes the result back so the work + // happens once per config change instead of on every read. + private void reparseStale(List stale, long currentRevision) { + SQLiteDatabase db = getWritableDatabase(); + db.beginTransaction(); + try { + for (Candidate candidate : stale) { + Candidate reparsed = ExpenseParser.parse( + appContext, + candidate.packageName, candidate.appName, candidate.notificationKey, + candidate.postedAt, candidate.title, candidate.text); + ContentValues values = new ContentValues(); + if (reparsed != null) { + candidate.merchant = reparsed.merchant; + candidate.amount = reparsed.amount; + candidate.currency = reparsed.currency; + candidate.originalAmount = reparsed.originalAmount; + candidate.originalCurrency = reparsed.originalCurrency; + candidate.suggestedCategory = reparsed.suggestedCategory; + candidate.suggestedPaymentMethod = reparsed.suggestedPaymentMethod; + candidate.note = reparsed.note; + candidate.transactionType = reparsed.transactionType; + candidate.postedAt = reparsed.postedAt; + values.put("merchant", value(candidate.merchant)); + values.put("amount", value(candidate.amount)); + values.put("currency", value(candidate.currency)); + values.put("original_amount", value(candidate.originalAmount)); + values.put("original_currency", value(candidate.originalCurrency)); + values.put("suggested_category", value(candidate.suggestedCategory)); + values.put("suggested_payment_method", value(candidate.suggestedPaymentMethod)); + values.put("note", value(candidate.note)); + values.put("transaction_type", value(candidate.transactionType, Candidate.TYPE_EXPENSE)); + values.put("posted_at", candidate.postedAt); + } + // A candidate the current config no longer matches keeps its stored + // fields; the revision still advances so it isn't retried until the + // config changes again. + values.put("parsed_revision", currentRevision); + db.update("candidates", values, "id = ?", new String[]{String.valueOf(candidate.id)}); } + db.setTransactionSuccessful(); + } finally { + db.endTransaction(); } - return items; } private Candidate fromCursor(Cursor cursor) { @@ -141,36 +218,12 @@ private Candidate fromCursor(Cursor cursor) { candidate.suggestedPaymentMethod = cursor.getString(cursor.getColumnIndexOrThrow("suggested_payment_method")); candidate.note = optionalString(cursor, "note"); String type = optionalString(cursor, "transaction_type"); - candidate.transactionType = type.isEmpty() ? "EXPENSE" : type; + candidate.transactionType = type.isEmpty() ? Candidate.TYPE_EXPENSE : type; candidate.postedAt = cursor.getLong(cursor.getColumnIndexOrThrow("posted_at")); candidate.status = cursor.getString(cursor.getColumnIndexOrThrow("status")); - reparseFromStoredSms(candidate); return candidate; } - // Re-derive parsed fields from the stored SMS so existing candidates reflect the - // current parser (payee, payment method, note, category, amount, transaction type). - // Identity (id, notification key, raw title/body) and workflow status are preserved. - private void reparseFromStoredSms(Candidate candidate) { - Candidate reparsed = ExpenseParser.parse( - appContext, - candidate.packageName, candidate.appName, candidate.notificationKey, - candidate.postedAt, candidate.title, candidate.text); - if (reparsed == null) { - return; - } - candidate.merchant = reparsed.merchant; - candidate.amount = reparsed.amount; - candidate.currency = reparsed.currency; - candidate.originalAmount = reparsed.originalAmount; - candidate.originalCurrency = reparsed.originalCurrency; - candidate.suggestedCategory = reparsed.suggestedCategory; - candidate.suggestedPaymentMethod = reparsed.suggestedPaymentMethod; - candidate.note = reparsed.note; - candidate.transactionType = reparsed.transactionType; - candidate.postedAt = reparsed.postedAt; - } - private static String value(String text) { return value(text, ""); } diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/ConfigActivity.java b/android_app/app/src/main/java/dev/fanis/expensenotification/ConfigActivity.java index 89d0c06..0000bc3 100644 --- a/android_app/app/src/main/java/dev/fanis/expensenotification/ConfigActivity.java +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/ConfigActivity.java @@ -18,9 +18,7 @@ import org.json.JSONObject; -import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; @@ -432,9 +430,9 @@ private String readConfig(String dirName, String name) { File user = userFile(dirName, name); try { if (user.exists()) { - return decodeUnicodeEscapes(readFile(user)); + return ConfigNames.decodeUnicodeEscapes(IoUtil.readFile(user)); } - return decodeUnicodeEscapes(readAsset(dirName + "/" + name)); + return ConfigNames.decodeUnicodeEscapes(IoUtil.readAsset(getAssets(), dirName + "/" + name)); } catch (IOException e) { return "{}"; } @@ -483,11 +481,7 @@ private String text(EditText input) { } private String cleanJsonName(String name) { - String cleaned = name == null ? "config.json" : name.trim().replaceAll("[\\\\/:*?\"<>|]", "-"); - if (cleaned.isEmpty()) { - cleaned = "config.json"; - } - return cleaned.endsWith(".json") ? cleaned : cleaned + ".json"; + return ConfigNames.cleanJsonName(name); } private String singular(String dirName) { @@ -531,49 +525,7 @@ private String displayName(Uri uri) { private String readUri(Uri uri) throws IOException { try (InputStream in = getContentResolver().openInputStream(uri)) { - return in == null ? "" : readAll(in); - } - } - - private String readAsset(String name) throws IOException { - try (InputStream in = getAssets().open(name)) { - return readAll(in); - } - } - - private String readFile(File file) throws IOException { - try (InputStream in = new FileInputStream(file)) { - return readAll(in); - } - } - - private String readAll(InputStream in) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - byte[] buffer = new byte[4096]; - int read; - while ((read = in.read(buffer)) != -1) { - out.write(buffer, 0, read); - } - return out.toString(StandardCharsets.UTF_8.name()); - } - - private String decodeUnicodeEscapes(String text) { - if (text == null || text.indexOf("\\u") < 0) { - return text; - } - StringBuilder out = new StringBuilder(text.length()); - for (int i = 0; i < text.length(); i++) { - if (i + 5 < text.length() && text.charAt(i) == '\\' && text.charAt(i + 1) == 'u') { - String hex = text.substring(i + 2, i + 6); - try { - out.append((char) Integer.parseInt(hex, 16)); - i += 5; - continue; - } catch (NumberFormatException ignored) { - } - } - out.append(text.charAt(i)); + return in == null ? "" : IoUtil.readAll(in); } - return out.toString(); } } diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/ConfigHides.java b/android_app/app/src/main/java/dev/fanis/expensenotification/ConfigHides.java index 5bc304c..0b4f749 100644 --- a/android_app/app/src/main/java/dev/fanis/expensenotification/ConfigHides.java +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/ConfigHides.java @@ -28,14 +28,6 @@ static boolean restore(Context context, String dirName, String name) { } private static File marker(Context context, String dirName, String name) { - return new File(new File(context.getFilesDir(), dirName), clean(name) + ".hidden"); - } - - private static String clean(String name) { - String cleaned = name == null ? "config.json" : name.trim().replaceAll("[\\\\/:*?\"<>|]", "-"); - if (cleaned.isEmpty()) { - cleaned = "config.json"; - } - return cleaned.endsWith(".json") ? cleaned : cleaned + ".json"; + return new File(new File(context.getFilesDir(), dirName), ConfigNames.cleanJsonName(name) + ".hidden"); } } diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/ConfigNames.java b/android_app/app/src/main/java/dev/fanis/expensenotification/ConfigNames.java new file mode 100644 index 0000000..52ad237 --- /dev/null +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/ConfigNames.java @@ -0,0 +1,64 @@ +package dev.fanis.expensenotification; + +/** Normalizes user-supplied config file names into safe ".json" form. */ +final class ConfigNames { + private ConfigNames() { + } + + static String cleanJsonName(String name) { + String cleaned = name == null ? "config.json" : name.trim().replaceAll("[\\\\/:*?\"<>|]", "-"); + if (cleaned.isEmpty()) { + cleaned = "config.json"; + } + return cleaned.endsWith(".json") ? cleaned : cleaned + ".json"; + } + + // Decodes backslash-u-XXXX escapes so bundled configs (kept ASCII-safe in the + // repo) show their Greek text readably in the editor. An escape whose backslash + // is itself escaped (double backslash before the 'u', e.g. a regex-level unicode + // escape inside a JSON string) is literal text and must be left untouched. + static String decodeUnicodeEscapes(String text) { + if (text == null || text.indexOf("\\u") < 0) { + return text; + } + StringBuilder out = new StringBuilder(text.length()); + int i = 0; + while (i < text.length()) { + char c = text.charAt(i); + if (c != '\\') { + out.append(c); + i++; + continue; + } + int j = i; + while (j < text.length() && text.charAt(j) == '\\') { + j++; + } + int run = j - i; + // Only an odd trailing backslash can start a real unicode escape. + if (run % 2 == 1 && j + 4 < text.length() && text.charAt(j) == 'u' + && isHex(text, j + 1, j + 5)) { + for (int k = 0; k < run - 1; k++) { + out.append('\\'); + } + out.append((char) Integer.parseInt(text.substring(j + 1, j + 5), 16)); + i = j + 5; + continue; + } + for (int k = 0; k < run; k++) { + out.append('\\'); + } + i = j; + } + return out.toString(); + } + + private static boolean isHex(String text, int from, int to) { + for (int i = from; i < to; i++) { + if (Character.digit(text.charAt(i), 16) < 0) { + return false; + } + } + return true; + } +} diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/ConfigValidator.java b/android_app/app/src/main/java/dev/fanis/expensenotification/ConfigValidator.java index 76fc69d..957ae02 100644 --- a/android_app/app/src/main/java/dev/fanis/expensenotification/ConfigValidator.java +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/ConfigValidator.java @@ -126,7 +126,7 @@ private static void validateRule(JSONObject rule, int index, Result result) { result.error(prefix + " regex rule is missing pattern"); } else { try { - Pattern.compile(pattern, flags(rule.optJSONArray("flags"))); + Pattern.compile(pattern, JsonValues.patternFlags(rule.optJSONArray("flags"))); } catch (PatternSyntaxException e) { result.error(prefix + " regex does not compile: " + e.getDescription()); } @@ -198,26 +198,6 @@ private static void validateScalar(String value, Set allowed, String pat } } - private static int flags(JSONArray json) { - int flags = 0; - if (json == null) { - return flags; - } - for (int i = 0; i < json.length(); i++) { - String flag = json.optString(i); - if ("caseInsensitive".equals(flag)) { - flags |= Pattern.CASE_INSENSITIVE; - } else if ("unicodeCase".equals(flag)) { - flags |= Pattern.UNICODE_CASE; - } else if ("multiline".equals(flag)) { - flags |= Pattern.MULTILINE; - } else if ("dotall".equals(flag)) { - flags |= Pattern.DOTALL; - } - } - return flags; - } - private static Set set(String... values) { HashSet set = new HashSet<>(); for (String value : values) { diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/ConfiguredParser.java b/android_app/app/src/main/java/dev/fanis/expensenotification/ConfiguredParser.java new file mode 100644 index 0000000..65c0ebc --- /dev/null +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/ConfiguredParser.java @@ -0,0 +1,204 @@ +package dev.fanis.expensenotification; + +import android.content.Context; +import android.content.res.AssetManager; + +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * The runtime parser: global config plus the enabled input sources, in priority + * order. The bundled asset configs are the single source of truth for defaults; + * user files in filesDir/inputs override assets of the same name, and hidden + * bundled configs are excluded entirely. Instances are immutable and cached per + * config revision, so rebuilding only happens after a config change. + */ +final class ConfiguredParser { + private static ConfiguredParser cachedContextParser; + private static long cachedRevision = Long.MIN_VALUE; + private static String cachedFilesDir = ""; + + private final GlobalConfig global; + private final List sources; + + private ConfiguredParser(GlobalConfig global, List sources) { + this.global = global; + this.sources = sources; + } + + /** Last-resort parser with no input sources; only used when a context is missing or loading fails. */ + static ConfiguredParser defaults() { + return new ConfiguredParser(GlobalConfig.defaults(), new ArrayList<>()); + } + + static ConfiguredParser forContext(Context context) { + if (context == null) { + return defaults(); + } + Context appContext = context.getApplicationContext(); + long revision = ConfigRevision.current(appContext); + String filesDir = appContext.getFilesDir() == null ? "" : appContext.getFilesDir().getAbsolutePath(); + synchronized (ConfiguredParser.class) { + if (cachedContextParser != null && cachedRevision == revision && cachedFilesDir.equals(filesDir)) { + return cachedContextParser; + } + } + try { + ConfiguredParser parser = fromContext(appContext); + synchronized (ConfiguredParser.class) { + cachedContextParser = parser; + cachedRevision = revision; + cachedFilesDir = filesDir; + } + return parser; + } catch (Exception ignored) { + return defaults(); + } + } + + boolean isWatched(String packageName, String title) { + for (InputSource source : sources) { + if (source.enabled && source.matches(packageName, title, global.smsPackages)) { + return true; + } + } + return false; + } + + Candidate parse(String packageName, String appName, String key, long postedAt, String title, String body) { + NotificationText text = combinedText(title, body); + if (isRejected(text.combined)) { + return null; + } + for (InputSource source : sources) { + if (!source.enabled || !source.matches(packageName, title, global.smsPackages)) { + continue; + } + for (InputRule rule : source.rules) { + Candidate candidate = rule.parse(global, packageName, appName, key, postedAt, title, body, text); + if (candidate != null) { + return candidate; + } + } + } + return null; + } + + ExpenseParser.ParseDiagnostics debug(String packageName, String appName, String key, long postedAt, + String title, String body) { + ArrayList steps = new ArrayList<>(); + NotificationText text = combinedText(title, body); + if (isRejected(text.combined)) { + steps.add("Rejected by global reject phrase."); + return new ExpenseParser.ParseDiagnostics(null, "", "", steps); + } + for (InputSource source : sources) { + String sourceName = source.displayName + " (" + source.id + ")"; + if (!source.enabled) { + steps.add("Skipped disabled source: " + sourceName); + continue; + } + if (!source.matches(packageName, title, global.smsPackages)) { + steps.add("Source did not match app/sender: " + sourceName); + continue; + } + steps.add("Source matched app/sender: " + sourceName); + for (InputRule rule : source.rules) { + Candidate candidate = rule.parse(global, packageName, appName, key, postedAt, title, body, text); + if (candidate != null) { + steps.add("Rule matched: " + rule.name + " (" + rule.type + ")"); + return new ExpenseParser.ParseDiagnostics(candidate, source.displayName, rule.name, steps); + } + steps.add("Rule did not match: " + rule.name + " (" + rule.type + ")"); + } + } + return new ExpenseParser.ParseDiagnostics(null, "", "", steps); + } + + private static NotificationText combinedText(String title, String body) { + return new NotificationText((ExpenseParser.safe(title) + "\n" + ExpenseParser.safe(body)).trim()); + } + + private boolean isRejected(String combined) { + String lower = combined.toLowerCase(Locale.ROOT); + for (String phrase : global.rejectPhrases) { + if (lower.contains(phrase)) { + return true; + } + } + return false; + } + + private static ConfiguredParser fromContext(Context context) throws JSONException { + AssetManager assets = context.getAssets(); + GlobalConfig global = GlobalConfig.defaults(); + try { + String assetGlobal = IoUtil.readAsset(assets, "global.json"); + if (!assetGlobal.isEmpty()) { + global = GlobalConfig.fromJson(new JSONObject(assetGlobal)); + } + } catch (IOException ignored) { + } + + LinkedHashMap inputs = new LinkedHashMap<>(); + try { + String[] names = assets.list("inputs"); + if (names != null) { + ArrayList sorted = new ArrayList<>(); + Collections.addAll(sorted, names); + Collections.sort(sorted); + for (String name : sorted) { + if (name.endsWith(".json") && !ConfigHides.isHidden(context, "inputs", name)) { + try { + inputs.put(name, IoUtil.readAsset(assets, "inputs/" + name)); + } catch (IOException ignored) { + } + } + } + } + } catch (IOException ignored) { + } + + File userInputs = new File(context.getFilesDir(), "inputs"); + File[] files = userInputs.listFiles(); + if (files != null) { + ArrayList sorted = new ArrayList<>(); + Collections.addAll(sorted, files); + Collections.sort(sorted, Comparator.comparing(File::getName)); + for (File file : sorted) { + if (file.isFile() && file.getName().endsWith(".json")) { + try { + inputs.put(file.getName(), IoUtil.readFile(file)); + } catch (IOException ignored) { + } + } + } + } + return fromJson(global.withSmsPackages(SmsApps.confirmedPackages(context)), inputs); + } + + static ConfiguredParser fromJson(String globalJson, LinkedHashMap inputJsonByFile) + throws JSONException { + return fromJson(GlobalConfig.fromJson(new JSONObject(globalJson)), inputJsonByFile); + } + + static ConfiguredParser fromJson(GlobalConfig global, LinkedHashMap inputJsonByFile) + throws JSONException { + ArrayList sources = new ArrayList<>(); + for (Map.Entry entry : inputJsonByFile.entrySet()) { + sources.add(InputSource.fromJson(entry.getKey(), new JSONObject(entry.getValue()))); + } + Collections.sort(sources, Comparator.comparingInt(source -> source.priority)); + return new ConfiguredParser(global, sources); + } +} diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/ExpenseEntryAccessibilityService.java b/android_app/app/src/main/java/dev/fanis/expensenotification/ExpenseEntryAccessibilityService.java index 674e469..3215354 100644 --- a/android_app/app/src/main/java/dev/fanis/expensenotification/ExpenseEntryAccessibilityService.java +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/ExpenseEntryAccessibilityService.java @@ -1,6 +1,7 @@ package dev.fanis.expensenotification; import android.accessibilityservice.AccessibilityService; +import android.accessibilityservice.AccessibilityServiceInfo; import android.content.SharedPreferences; import android.os.Bundle; import android.view.accessibility.AccessibilityEvent; @@ -25,7 +26,51 @@ public class ExpenseEntryAccessibilityService extends AccessibilityService { private static final String STATE_AWAIT_SAVE = "AWAIT_SAVE"; private static final String STATE_FILLED = "FILLED"; + // A fill the user abandoned must not leave the service inspecting the target + // app's windows forever; after this long any pending state is discarded. + private static final long STATE_TTL_MS = 15 * 60 * 1000L; + private long lastAttemptAt; + // Strong reference: SharedPreferences only holds listeners weakly. + private SharedPreferences.OnSharedPreferenceChangeListener targetPackageListener; + + // Restrict event delivery to the configured output app. Without this filter the + // service receives window/content events from every app on the device and pays + // the cross-process wakeup cost for each; with it the system drops everything + // else before it reaches us. Re-applied whenever the target package changes. + @Override + protected void onServiceConnected() { + super.onServiceConnected(); + applyPackageFilter(); + targetPackageListener = (prefs, key) -> { + if ("target_package".equals(key)) { + applyPackageFilter(); + } + }; + getSharedPreferences("automation", MODE_PRIVATE) + .registerOnSharedPreferenceChangeListener(targetPackageListener); + } + + @Override + public void onDestroy() { + if (targetPackageListener != null) { + getSharedPreferences("automation", MODE_PRIVATE) + .unregisterOnSharedPreferenceChangeListener(targetPackageListener); + targetPackageListener = null; + } + super.onDestroy(); + } + + private void applyPackageFilter() { + AccessibilityServiceInfo info = getServiceInfo(); + if (info == null) { + return; + } + info.packageNames = new String[]{ + getSharedPreferences("automation", MODE_PRIVATE) + .getString("target_package", EXPENSE_MANAGER_PACKAGE)}; + setServiceInfo(info); + } @Override public void onAccessibilityEvent(AccessibilityEvent event) { @@ -39,6 +84,10 @@ public void onAccessibilityEvent(AccessibilityEvent event) { } String state = prefs.getString("state", ""); + if (isExpired(prefs, state)) { + prefs.edit().putString("state", "").apply(); + return; + } // While waiting for the user to finish, keep a live copy of the payee they // have chosen, and commit the alias both when they tap OK and (as a fallback, @@ -101,9 +150,18 @@ private void prefillPayeeAndAwaitSave(SharedPreferences prefs, AccessibilityNode prefs.edit() .putString("prefill_text", prefillText) .putString("state", STATE_AWAIT_SAVE) + .putLong("state_at", System.currentTimeMillis()) .apply(); } + private static boolean isExpired(SharedPreferences prefs, String state) { + if (!STATE_PENDING.equals(state) && !STATE_AWAIT_SAVE.equals(state)) { + return false; + } + long stateAt = prefs.getLong("state_at", 0L); + return stateAt > 0 && System.currentTimeMillis() - stateAt > STATE_TTL_MS; + } + private boolean isSaveClick(AccessibilityEvent event) { AccessibilityNodeInfo source = event.getSource(); if (source == null) { diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/ExpenseNotificationListener.java b/android_app/app/src/main/java/dev/fanis/expensenotification/ExpenseNotificationListener.java index d7c2154..18ae399 100644 --- a/android_app/app/src/main/java/dev/fanis/expensenotification/ExpenseNotificationListener.java +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/ExpenseNotificationListener.java @@ -11,9 +11,17 @@ import android.service.notification.StatusBarNotification; import java.lang.ref.WeakReference; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; public class ExpenseNotificationListener extends NotificationListenerService { private static WeakReference instance = new WeakReference<>(null); + // Keeps parsing and database writes off the process main thread, which also + // serializes them so a burst of notifications can't interleave. + private static final ExecutorService WORKER = Executors.newSingleThreadExecutor(); public static int scanActive(Context context) { ExpenseNotificationListener listener = instance.get(); @@ -27,11 +35,6 @@ public static boolean isConnected() { return instance.get() != null; } - @Override - public void onCreate() { - super.onCreate(); - } - private boolean isWatched(String packageName, String title) { return ExpenseParser.isWatched(this, packageName, title); } @@ -42,7 +45,7 @@ public void onListenerConnected() { diagnostics(this).edit() .putLong("last_connected_at", System.currentTimeMillis()) .apply(); - scanActiveNotifications(this); + WORKER.execute(() -> scanActiveNotifications(this)); } @Override @@ -61,7 +64,7 @@ public void onListenerDisconnected() { @Override public void onNotificationPosted(StatusBarNotification sbn) { - saveCandidateIfRelevant(this, sbn); + WORKER.execute(() -> saveCandidateIfRelevant(this, sbn)); } private int scanActiveNotifications(Context context) { @@ -115,32 +118,28 @@ private boolean saveCandidateIfRelevant(Context context, StatusBarNotification s String tag = sbn.getTag(); String ticker = notification.tickerText == null ? "" : notification.tickerText.toString(); String body = join(text, bigText, subText, ticker, tag); - diagnostics(context).edit() + + SharedPreferences.Editor diagnostics = diagnostics(context).edit() .putLong("last_watched_at", System.currentTimeMillis()) .putString("last_watched_package", sbn.getPackageName()) .putString("last_watched_title", title) - .putString("last_watched_body", body) - .apply(); + .putString("last_watched_body", body); Candidate candidate = ExpenseParser.parse( context, sbn.getPackageName(), appName(context, sbn.getPackageName()), - dedupeKey(sbn, body), + dedupeKey(sbn.getKey(), body), sbn.getPostTime(), title.isEmpty() ? firstNonEmpty(ticker, tag) : title, body); if (candidate == null) { - diagnostics(context).edit() - .putString("last_result", "Parser rejected watched notification") - .apply(); + diagnostics.putString("last_result", "Parser rejected watched notification").apply(); return false; } - long inserted = new CandidateDb(context).insertIfNew(candidate); - diagnostics(context).edit() - .putString("last_result", inserted == -1 ? "Duplicate candidate ignored" : "Candidate saved") - .apply(); + long inserted = CandidateDb.getInstance(context).insertIfNew(candidate); + diagnostics.putString("last_result", inserted == -1 ? "Duplicate candidate ignored" : "Candidate saved").apply(); return inserted != -1; } @@ -152,8 +151,24 @@ private boolean saveCandidateIfRelevant(Context context, StatusBarNotification s // separate candidate, while re-scanning the same still-active notification (same key // + same body) still dedupes. Per-notification sources like Revolut keep a unique // sbn.getKey() per transaction, so two identical charges remain two candidates. - private static String dedupeKey(StatusBarNotification sbn, String body) { - return sbn.getKey() + "#" + Integer.toHexString((body == null ? "" : body).hashCode()); + // SHA-256 (not String.hashCode) so two different SMS can't collide into one key + // and silently drop an expense. + static String dedupeKey(String sbnKey, String body) { + return sbnKey + "#" + sha256(body == null ? "" : body); + } + + private static String sha256(String text) { + try { + byte[] digest = MessageDigest.getInstance("SHA-256").digest(text.getBytes(StandardCharsets.UTF_8)); + StringBuilder hex = new StringBuilder(digest.length * 2); + for (byte b : digest) { + hex.append(Character.forDigit((b >> 4) & 0xf, 16)).append(Character.forDigit(b & 0xf, 16)); + } + return hex.toString(); + } catch (NoSuchAlgorithmException e) { + // Every Android ships SHA-256; keep a deterministic fallback anyway. + return Integer.toHexString(text.hashCode()); + } } private static String titleOf(StatusBarNotification sbn) { diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/ExpenseParser.java b/android_app/app/src/main/java/dev/fanis/expensenotification/ExpenseParser.java index 7165c96..bfced7e 100644 --- a/android_app/app/src/main/java/dev/fanis/expensenotification/ExpenseParser.java +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/ExpenseParser.java @@ -1,41 +1,38 @@ package dev.fanis.expensenotification; import android.content.Context; -import android.content.res.AssetManager; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Calendar; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.json.JSONArray; import org.json.JSONException; -import org.json.JSONObject; +/** + * Entry points for turning a notification into an expense {@link Candidate}, plus + * the language-level heuristics (amounts, merchants, categories, Greek folding) + * shared by every input source. The config-driven matching machinery lives in + * {@link ConfiguredParser}/{@link InputSource}/{@link InputRule}. + */ final class ExpenseParser { // Default payment method for card purchases whose specific card can't be // identified. Must match an existing payment-method name in Expense Manager, // otherwise the prefill intent leaves the field blank. - private static final String DEFAULT_PAYMENT_METHOD = "Credit Card"; + static final String DEFAULT_PAYMENT_METHOD = "Credit Card"; - // A money amount with optional grouped thousands (1.234,56) or a plain value (300,00 / 12.34). - // normalizeAmount() resolves which separator is the decimal point afterwards. - private static final String MONEY = "(?:[0-9]{1,3}(?:[.,][0-9]{3})*[.,][0-9]{2}|[0-9]+(?:[.,][0-9]{2})?)"; + // A money amount: grouped thousands with decimals (1.234,56), grouped thousands + // without decimals (1.234 = one thousand two hundred thirty four), or a plain + // value (300,00 / 12.34 / 1234). normalizeAmount() resolves which separator is + // the decimal point afterwards. + static final String MONEY = + "(?:[0-9]{1,3}(?:[.,][0-9]{3})+[.,][0-9]{2}(?![0-9])" + + "|[0-9]{1,3}(?:[.,][0-9]{3})+(?![0-9])" + + "|[0-9]+(?:[.,][0-9]{2})?)"; // The transaction date the bank wrote into the SMS body (Cyprus/EU day-first: // dd/MM/yyyy or dd/MM/yy, optional HH:mm after a space or comma). Used to date the @@ -66,29 +63,6 @@ final class ExpenseParser { private ExpenseParser() { } - // Notifications that look payment-shaped but are not a completed charge: 3DS - // approval prompts (the duplicate that precedes a real "successful" message), - // declines, and card verification/registration. Matched as whole phrases so they - // don't catch a genuine payment whose merchant name happens to contain a word. - private static final String[] REJECT_PHRASES = { - "waiting for your approval", - "verify a payment", - "verify your payment", - "tap to start", - "declined", - "insufficient balance", - "insufficient funds", - "verified your card", - "haven't been charged", - "have not been charged", - "card registration", - "card verification", - }; - - static Candidate parse(String packageName, String appName, String key, long postedAt, String title, String body) { - return ConfiguredParser.defaults().parse(packageName, appName, key, postedAt, title, body); - } - static Candidate parse(Context context, String packageName, String appName, String key, long postedAt, String title, String body) { return ConfiguredParser.forContext(context).parse(packageName, appName, key, postedAt, title, body); } @@ -101,16 +75,19 @@ static ParseDiagnostics debug(Context context, String packageName, String appNam return ConfiguredParser.forContext(context).debug(packageName, appName, key, postedAt, title, body); } + /** Parses against a single input config with the default global config (embedded config tests). */ static Candidate parseInputConfig(String fileName, String inputJson, String packageName, String appName, String key, long postedAt, String title, String body) throws JSONException { LinkedHashMap inputs = new LinkedHashMap<>(); inputs.put(fileName == null || fileName.isEmpty() ? "input.json" : fileName, inputJson); - return ConfiguredParser.fromJson(DefaultJson.global(), inputs).parse(packageName, appName, key, postedAt, title, body); + return ConfiguredParser.fromJson(GlobalConfig.defaultJson(), inputs).parse(packageName, appName, key, postedAt, title, body); } - static String defaultInputJson(String fileName) { - String json = DefaultJson.inputsByFile().get(fileName); - return json == null ? "" : json; + /** Parses against an explicit global + input set; lets JVM tests run the real asset configs. */ + static Candidate parseConfigured(String globalJson, LinkedHashMap inputJsonByFile, + String packageName, String appName, String key, long postedAt, + String title, String body) throws JSONException { + return ConfiguredParser.fromJson(globalJson, inputJsonByFile).parse(packageName, appName, key, postedAt, title, body); } static Set defaultSmsPackages() { @@ -135,17 +112,7 @@ boolean matched() { } } - private static boolean isRejectedNotification(String combined) { - String lower = safe(combined).toLowerCase(Locale.ROOT); - for (String phrase : REJECT_PHRASES) { - if (lower.contains(phrase)) { - return true; - } - } - return false; - } - - private static boolean isZero(String value) { + static boolean isZero(String value) { try { return new BigDecimal(value).signum() == 0; } catch (Exception ignored) { @@ -153,7 +120,7 @@ private static boolean isZero(String value) { } } - private static boolean isInteresting(Candidate candidate) { + static boolean isInteresting(Candidate candidate) { if (candidate.hasAmount()) { return true; } @@ -161,7 +128,7 @@ private static boolean isInteresting(Candidate candidate) { return haystack.contains("payment") || haystack.contains("purchase") || haystack.contains("card"); } - private static Amount extractAmount(String text) { + static Amount extractAmount(String text) { Matcher matcher = SYMBOL_AMOUNT.matcher(text); if (matcher.find()) { return new Amount(currencyFromSymbol(matcher.group(1)), normalizeAmount(matcher.group(2))); @@ -181,7 +148,7 @@ private static Amount extractAmount(String text) { return null; } - private static MultiCurrencyAmount extractMultiCurrencyAmount(String text) { + static MultiCurrencyAmount extractMultiCurrencyAmount(String text) { Matcher matcher = MULTI_CURRENCY_PAID.matcher(text); if (!matcher.find()) { return null; @@ -208,20 +175,20 @@ private static String accountCurrencyFromBody(String text) { return matcher.find() ? matcher.group(1).toUpperCase(Locale.ROOT) : null; } - private static String accountDebitCategory(String description) { + static String accountDebitCategory(String description) { String folded = foldAmbiguousGreek(description).toUpperCase(Locale.ROOT); - if (folded.contains("PROM") || folded.contains("\u03A0POM") - || folded.contains(foldAmbiguousGreek("\u03A0\u03A1\u039F\u039C\u0397\u0398\u0395\u0399\u0391").toUpperCase(Locale.ROOT))) { + if (folded.contains("PROM") || folded.contains("ΠPOM") + || folded.contains(foldAmbiguousGreek("ΠΡΟΜΗΘΕΙΑ").toUpperCase(Locale.ROOT))) { return "Bank fees"; } - if (folded.contains("META") || folded.contains("\u03A6OPA") - || folded.contains(foldAmbiguousGreek("\u039C\u0395\u03A4\u0391\u03A6\u039F\u03A1\u0391").toUpperCase(Locale.ROOT))) { + if (folded.contains("META") || folded.contains("ΦOPA") + || folded.contains(foldAmbiguousGreek("ΜΕΤΑΦΟΡΑ").toUpperCase(Locale.ROOT))) { return "Transfers"; } return categoryFor(description); } - private static String guessMerchant(String title, String body, Amount amount) { + static String guessMerchant(String title, String body, Amount amount) { String combined = safe(title) + "\n" + safe(body); Matcher paidAt = PAID_AT.matcher(combined); if (paidAt.find()) { @@ -252,7 +219,7 @@ private static String guessMerchant(String title, String body, Amount amount) { return isNonMerchantLine(fallback) ? "Unknown" : fallback; } - private static String cleanupMerchant(String merchant) { + static String cleanupMerchant(String merchant) { String cleaned = safe(merchant).trim().replaceAll("\\s+", " "); int balanceIndex = cleaned.toLowerCase(Locale.ROOT).indexOf(" eur balance"); if (balanceIndex >= 0) { @@ -271,7 +238,7 @@ private static boolean isNonMerchantLine(String line) { isMaskedWalletDetail(line); } - private static String categoryFor(String merchant) { + static String categoryFor(String merchant) { String lower = safe(merchant).toLowerCase(Locale.ROOT); if (lower.contains("alphamega") || lower.contains("supermarket") || lower.contains("lidl") || lower.contains("metro") || lower.contains("sklavenitis")) { @@ -288,7 +255,7 @@ private static String categoryFor(String merchant) { return ""; } - private static String paymentMethodFor(String packageName, String appName, String notificationText) { + static String paymentMethodFor(String packageName, String appName, String notificationText) { String lowerPackage = safe(packageName).toLowerCase(Locale.ROOT); String lowerApp = safe(appName).toLowerCase(Locale.ROOT); if (lowerPackage.contains("revolut") || lowerApp.contains("revolut")) { @@ -306,7 +273,7 @@ private static String paymentMethodFor(String packageName, String appName, Strin return DEFAULT_PAYMENT_METHOD; } - private static String walletPaymentMethod(String notificationText) { + static String walletPaymentMethod(String notificationText) { Matcher matcher = WALLET_PAYMENT_METHOD.matcher(safe(notificationText)); if (!matcher.find()) { return ""; @@ -326,7 +293,7 @@ private static boolean isGenericCardName(String name) { boolean hasBrand = lower.contains("visa") || lower.contains("mastercard") || lower.contains("maestro") || lower.contains("amex") || lower.contains("american express"); - boolean hasMask = name.indexOf('*') >= 0 || name.indexOf('\u2022') >= 0; + boolean hasMask = name.indexOf('*') >= 0 || name.indexOf('•') >= 0; return hasBrand || hasMask; } @@ -336,13 +303,13 @@ private static boolean isMaskedWalletDetail(String line) { } private static String currencyFromSymbol(String symbol) { - if ("\u20AC".equals(symbol)) { + if ("€".equals(symbol)) { return "EUR"; } if ("$".equals(symbol)) { return "USD"; } - if ("\u00A3".equals(symbol)) { + if ("£".equals(symbol)) { return "GBP"; } return symbol; @@ -352,7 +319,7 @@ private static String currencyFromSymbol(String symbol) { // falling back to when the notification was posted. Time of day is kept when the // message includes it, otherwise noon (a safe hour that never rolls the calendar // day over across time zones). - private static long transactionTime(String title, String body, long fallback) { + static long transactionTime(String title, String body, long fallback) { Matcher matcher = TRANSACTION_DATE.matcher(safe(title) + "\n" + safe(body)); if (!matcher.find()) { return fallback; @@ -387,7 +354,7 @@ private static long transactionTime(String title, String body, long fallback) { } } - private static String normalizeAmount(String raw) { + static String normalizeAmount(String raw) { if (raw == null) { return ""; } @@ -402,8 +369,16 @@ private static String normalizeAmount(String raw) { s = s.replace(",", ""); } } else if (lastComma >= 0) { - // Single comma: decimal separator (el-CY / el-GR style). - s = s.replace(',', '.'); + // Single comma: exactly 3 digits after it is a thousands group (1,234); + // otherwise it's an el-CY / el-GR decimal separator (300,00). + if (s.length() - lastComma - 1 == 3) { + s = s.replace(",", ""); + } else { + s = s.replace(',', '.'); + } + } else if (lastDot >= 0 && s.length() - lastDot - 1 == 3) { + // Single dot with exactly 3 digits after it: European thousands (1.234). + s = s.replace(".", ""); } return s; } @@ -412,7 +387,7 @@ private static String normalizeAmount(String raw) { // twin. 1:1 and length-preserving, so match offsets stay valid against the // original string. Greek letters with no Latin lookalike are left as-is and // serve as unambiguous anchors in the patterns. - private static String foldAmbiguousGreek(String text) { + static String foldAmbiguousGreek(String text) { if (text == null) { return ""; } @@ -425,737 +400,29 @@ private static String foldAmbiguousGreek(String text) { private static char foldGreekChar(char c) { switch (c) { - case '\u0391': return 'A'; // Alpha - case '\u0392': return 'B'; // Beta - case '\u0395': return 'E'; // Epsilon - case '\u0396': return 'Z'; // Zeta - case '\u0397': return 'H'; // Eta - case '\u0399': return 'I'; // Iota - case '\u039A': return 'K'; // Kappa - case '\u039C': return 'M'; // Mu - case '\u039D': return 'N'; // Nu - case '\u039F': return 'O'; // Omicron - case '\u03A1': return 'P'; // Rho - case '\u03A4': return 'T'; // Tau - case '\u03A5': return 'Y'; // Upsilon - case '\u03A7': return 'X'; // Chi + case 'Α': return 'A'; // Alpha + case 'Β': return 'B'; // Beta + case 'Ε': return 'E'; // Epsilon + case 'Ζ': return 'Z'; // Zeta + case 'Η': return 'H'; // Eta + case 'Ι': return 'I'; // Iota + case 'Κ': return 'K'; // Kappa + case 'Μ': return 'M'; // Mu + case 'Ν': return 'N'; // Nu + case 'Ο': return 'O'; // Omicron + case 'Ρ': return 'P'; // Rho + case 'Τ': return 'T'; // Tau + case 'Υ': return 'Y'; // Upsilon + case 'Χ': return 'X'; // Chi default: return c; } } - private static String safe(String text) { + static String safe(String text) { return text == null ? "" : text; } - private static final class ConfiguredParser { - private static final ConfiguredParser DEFAULTS = fromDefaults(); - private static ConfiguredParser cachedContextParser; - private static long cachedRevision = Long.MIN_VALUE; - private static String cachedFilesDir = ""; - - private final GlobalConfig global; - private final List sources; - - private ConfiguredParser(GlobalConfig global, List sources) { - this.global = global; - this.sources = sources; - } - - static ConfiguredParser defaults() { - return DEFAULTS; - } - - static ConfiguredParser forContext(Context context) { - if (context == null) { - return defaults(); - } - Context appContext = context.getApplicationContext(); - long revision = ConfigRevision.current(appContext); - String filesDir = appContext.getFilesDir() == null ? "" : appContext.getFilesDir().getAbsolutePath(); - synchronized (ConfiguredParser.class) { - if (cachedContextParser != null && cachedRevision == revision && cachedFilesDir.equals(filesDir)) { - return cachedContextParser; - } - } - try { - ConfiguredParser parser = fromContext(appContext); - synchronized (ConfiguredParser.class) { - cachedContextParser = parser; - cachedRevision = revision; - cachedFilesDir = filesDir; - } - return parser; - } catch (Exception ignored) { - return defaults(); - } - } - - boolean isWatched(String packageName, String title) { - for (InputSource source : sources) { - if (source.enabled && source.matches(packageName, title, global.smsPackages)) { - return true; - } - } - return false; - } - - Candidate parse(String packageName, String appName, String key, long postedAt, String title, String body) { - String combined = (safe(title) + "\n" + safe(body)).trim(); - if (isRejected(combined)) { - return null; - } - for (InputSource source : sources) { - if (!source.enabled || !source.matches(packageName, title, global.smsPackages)) { - continue; - } - for (InputRule rule : source.rules) { - Candidate candidate = rule.parse(source, packageName, appName, key, postedAt, title, body, combined); - if (candidate != null) { - return candidate; - } - } - } - return null; - } - - ParseDiagnostics debug(String packageName, String appName, String key, long postedAt, String title, String body) { - ArrayList steps = new ArrayList<>(); - String combined = (safe(title) + "\n" + safe(body)).trim(); - if (isRejected(combined)) { - steps.add("Rejected by global reject phrase."); - return new ParseDiagnostics(null, "", "", steps); - } - for (InputSource source : sources) { - String sourceName = source.displayName + " (" + source.id + ")"; - if (!source.enabled) { - steps.add("Skipped disabled source: " + sourceName); - continue; - } - if (!source.matches(packageName, title, global.smsPackages)) { - steps.add("Source did not match app/sender: " + sourceName); - continue; - } - steps.add("Source matched app/sender: " + sourceName); - for (InputRule rule : source.rules) { - Candidate candidate = rule.parse(source, packageName, appName, key, postedAt, title, body, combined); - if (candidate != null) { - steps.add("Rule matched: " + rule.name + " (" + rule.type + ")"); - return new ParseDiagnostics(candidate, source.displayName, rule.name, steps); - } - steps.add("Rule did not match: " + rule.name + " (" + rule.type + ")"); - } - } - return new ParseDiagnostics(null, "", "", steps); - } - - private boolean isRejected(String combined) { - String lower = safe(combined).toLowerCase(Locale.ROOT); - for (String phrase : global.rejectPhrases) { - if (lower.contains(phrase)) { - return true; - } - } - return false; - } - - private static ConfiguredParser fromDefaults() { - try { - return fromJson(DefaultJson.global(), DefaultJson.inputsByFile()); - } catch (Exception ignored) { - return new ConfiguredParser(GlobalConfig.defaults(), new ArrayList<>()); - } - } - - private static ConfiguredParser fromContext(Context context) throws IOException, JSONException { - LinkedHashMap inputs = new LinkedHashMap<>(DefaultJson.inputsByFile()); - String globalJson = DefaultJson.global(); - AssetManager assets = context.getAssets(); - try { - String assetGlobal = readAsset(assets, "global.json"); - if (!assetGlobal.isEmpty()) { - globalJson = assetGlobal; - } - } catch (IOException ignored) { - } - try { - String[] names = assets.list("inputs"); - if (names != null) { - ArrayList sorted = new ArrayList<>(); - Collections.addAll(sorted, names); - Collections.sort(sorted); - for (String name : sorted) { - if (name.endsWith(".json") && !ConfigHides.isHidden(context, "inputs", name)) { - inputs.put(name, readAsset(assets, "inputs/" + name)); - } - } - } - } catch (IOException ignored) { - } - - File userInputs = new File(context.getFilesDir(), "inputs"); - File[] files = userInputs.listFiles(); - if (files != null) { - ArrayList sorted = new ArrayList<>(); - Collections.addAll(sorted, files); - Collections.sort(sorted, Comparator.comparing(File::getName)); - for (File file : sorted) { - if (file.isFile() && file.getName().endsWith(".json")) { - inputs.put(file.getName(), readFile(file)); - } - } - } - return fromJson(globalJson, inputs, SmsApps.confirmedPackages(context)); - } - - private static ConfiguredParser fromJson(String globalJson, LinkedHashMap inputJsonByFile) throws JSONException { - return fromJson(globalJson, inputJsonByFile, Collections.emptySet()); - } - - private static ConfiguredParser fromJson(String globalJson, LinkedHashMap inputJsonByFile, - Set extraSmsPackages) throws JSONException { - GlobalConfig global = GlobalConfig.fromJson(new JSONObject(globalJson)).withSmsPackages(extraSmsPackages); - ArrayList sources = new ArrayList<>(); - for (Map.Entry entry : inputJsonByFile.entrySet()) { - sources.add(InputSource.fromJson(entry.getKey(), new JSONObject(entry.getValue()))); - } - Collections.sort(sources, Comparator.comparingInt(source -> source.priority)); - return new ConfiguredParser(global, sources); - } - - private static String readAsset(AssetManager assets, String name) throws IOException { - try (InputStream in = assets.open(name)) { - return readAll(in); - } - } - - private static String readFile(File file) throws IOException { - try (InputStream in = new FileInputStream(file)) { - return readAll(in); - } - } - - private static String readAll(InputStream in) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - byte[] buffer = new byte[4096]; - int read; - while ((read = in.read(buffer)) != -1) { - out.write(buffer, 0, read); - } - return out.toString(StandardCharsets.UTF_8.name()); - } - } - - private static final class GlobalConfig { - final Set smsPackages; - final List rejectPhrases; - final boolean dropZeroAmount; - - private GlobalConfig(Set smsPackages, List rejectPhrases, boolean dropZeroAmount) { - this.smsPackages = smsPackages; - this.rejectPhrases = rejectPhrases; - this.dropZeroAmount = dropZeroAmount; - } - - static GlobalConfig defaults() { - return new GlobalConfig( - set("com.textra", "com.google.android.apps.messaging", "com.samsung.android.messaging", "com.android.mms"), - list(REJECT_PHRASES), - true); - } - - static GlobalConfig fromJson(JSONObject json) { - return new GlobalConfig( - setFromJson(json.optJSONArray("smsPackages"), defaults().smsPackages), - strings(json.optJSONArray("rejectPhrases"), defaults().rejectPhrases), - json.optBoolean("dropZeroAmount", true)); - } - - GlobalConfig withSmsPackages(Set extraSmsPackages) { - if (extraSmsPackages == null || extraSmsPackages.isEmpty()) { - return this; - } - HashSet merged = new HashSet<>(smsPackages); - merged.addAll(extraSmsPackages); - return new GlobalConfig(merged, rejectPhrases, dropZeroAmount); - } - } - - private static final class InputSource { - final String id; - final String displayName; - final boolean enabled; - final int priority; - final Set packages; - final Set senders; - final List transforms; - final List rules; - - private InputSource(String id, String displayName, boolean enabled, int priority, Set packages, - Set senders, List transforms, List rules) { - this.id = id; - this.displayName = displayName; - this.enabled = enabled; - this.priority = priority; - this.packages = packages; - this.senders = senders; - this.transforms = transforms; - this.rules = rules; - } - - static InputSource fromJson(String fileName, JSONObject json) { - JSONObject match = json.optJSONObject("match"); - List rules = new ArrayList<>(); - JSONArray rawRules = json.optJSONArray("rules"); - if (rawRules != null) { - for (int i = 0; i < rawRules.length(); i++) { - JSONObject rawRule = rawRules.optJSONObject(i); - if (rawRule != null) { - rules.add(InputRule.fromJson(rawRule)); - } - } - } - return new InputSource( - json.optString("id", stripJson(fileName)), - json.optString("displayName", stripJson(fileName)), - json.optBoolean("enabled", true), - json.optInt("priority", 100), - setFromJson(match == null ? null : match.optJSONArray("packages"), Collections.emptySet()), - lowerSet(strings(match == null ? null : match.optJSONArray("senders"), Collections.emptyList())), - strings(json.optJSONArray("transforms"), Collections.emptyList()), - rules); - } - - boolean matches(String packageName, String title, Set smsPackages) { - String safePackage = safe(packageName); - if (packages.contains(safePackage)) { - return true; - } - if (!senders.isEmpty() && smsPackages.contains(safePackage)) { - String lowerTitle = safe(title).trim().toLowerCase(Locale.ROOT); - return senders.contains(lowerTitle); - } - return packages.isEmpty() && senders.isEmpty(); - } - } - - private static final class InputRule { - final String name; - final String type; - final String pattern; - final int flags; - final List transforms; - final JSONObject output; - final String merchantSource; - final String noteSource; - - private InputRule(String name, String type, String pattern, int flags, List transforms, - JSONObject output, String merchantSource, String noteSource) { - this.name = name; - this.type = type; - this.pattern = pattern; - this.flags = flags; - this.transforms = transforms; - this.output = output == null ? new JSONObject() : output; - this.merchantSource = merchantSource; - this.noteSource = noteSource; - } - - static InputRule fromJson(JSONObject json) { - return new InputRule( - json.optString("name", ""), - json.optString("type", "regex"), - json.optString("pattern", ""), - flags(json.optJSONArray("flags")), - strings(json.optJSONArray("transforms"), Collections.emptyList()), - json.optJSONObject("output"), - json.optString("merchantSource", ""), - json.optString("noteSource", "")); - } - - Candidate parse(InputSource source, String packageName, String appName, String key, long postedAt, - String title, String body, String combined) { - if ("amountFallback".equals(type)) { - return amountFallback(source, packageName, appName, key, postedAt, title, body, combined); - } - if (!"regex".equals(type) || pattern.isEmpty()) { - return null; - } - return regexCandidate(source, packageName, appName, key, postedAt, title, body, combined); - } - - private Candidate amountFallback(InputSource source, String packageName, String appName, String key, long postedAt, - String title, String body, String combined) { - boolean multiCurrency = hasTransform(source, "multiCurrency"); - MultiCurrencyAmount multi = multiCurrency ? extractMultiCurrencyAmount(combined) : null; - Amount amount = multi != null ? multi.account : extractAmount(combined); - if (amount == null) { - return null; - } - if (isZero(amount.value)) { - return null; - } - Candidate candidate = baseCandidate(packageName, appName, key, postedAt, title, body); - candidate.amount = amount.value; - candidate.currency = amount.currency; - if (multi != null) { - candidate.originalAmount = multi.original.value; - candidate.originalCurrency = multi.original.currency; - } - candidate.merchant = merchantFromSource(merchantSource, title, body, amount); - candidate.note = "body".equals(noteSource) ? safe(body) : ""; - candidate.suggestedPaymentMethod = paymentMethod(combined, packageName, appName, output.optString("paymentMethod", "@packageDefault"), ""); - candidate.suggestedCategory = category(candidate.merchant, output.optString("category", "@keyword")); - candidate.status = "NEW"; - return isInteresting(candidate) ? candidate : null; - } - - private Candidate regexCandidate(InputSource source, String packageName, String appName, String key, long postedAt, - String title, String body, String combined) { - boolean foldGreek = hasTransform(source, "foldGreek") || transforms.contains("foldGreek"); - String target = foldGreek ? foldAmbiguousGreek(combined) : combined; - String rawPattern = foldGreek ? foldAmbiguousGreek(pattern) : pattern; - Matcher matcher = Pattern.compile(rawPattern, flags).matcher(target); - if (!matcher.find()) { - return null; - } - String amount = normalizeAmount(group(matcher, combined, "amount")); - if (amount.isEmpty()) { - return null; - } - if (isZero(amount)) { - return null; - } - Candidate candidate = baseCandidate(packageName, appName, key, postedAt, title, body); - candidate.amount = amount; - candidate.currency = valueOrGroup(matcher, combined, output.optString("currency", ""), "currency"); - candidate.originalAmount = ""; - candidate.originalCurrency = ""; - String merchant = group(matcher, combined, "merchant"); - if (merchant.isEmpty()) { - merchant = merchantFromSource(merchantSource, title, body, new Amount(candidate.currency, candidate.amount)); - } - candidate.merchant = cleanupMerchant(merchant); - candidate.note = group(matcher, combined, "note"); - if (candidate.note.isEmpty() && "body".equals(noteSource)) { - candidate.note = safe(body); - } - String card = group(matcher, combined, "card"); - candidate.suggestedPaymentMethod = paymentMethod(combined, packageName, appName, - output.optString("paymentMethod", "@packageDefault"), card); - String category = output.optString("category", "@keyword"); - candidate.suggestedCategory = category(candidate.merchant, category); - candidate.transactionType = output.optString("type", "EXPENSE"); - if ("INCOME".equals(candidate.transactionType) && candidate.suggestedCategory.isEmpty()) { - candidate.suggestedCategory = "Income"; - } - candidate.status = "NEW"; - return candidate; - } - - private boolean hasTransform(InputSource source, String name) { - return source.transforms.contains(name) || transforms.contains(name); - } - } - - private static Candidate baseCandidate(String packageName, String appName, String key, long postedAt, String title, String body) { - Candidate candidate = new Candidate(); - candidate.notificationKey = key; - candidate.packageName = packageName; - candidate.appName = appName; - candidate.title = safe(title); - candidate.text = safe(body); - candidate.postedAt = transactionTime(title, body, postedAt); - candidate.originalAmount = ""; - candidate.originalCurrency = ""; - candidate.note = ""; - candidate.transactionType = "EXPENSE"; - return candidate; - } - - private static String merchantFromSource(String merchantSource, String title, String body, Amount amount) { - if ("empty".equals(merchantSource)) { - return ""; - } - if ("title".equals(merchantSource)) { - return cleanupMerchant(title); - } - return guessMerchant(title, body, amount); - } - - private static String paymentMethod(String combined, String packageName, String appName, String template, String card) { - if ("@packageDefault".equals(template)) { - return paymentMethodFor(packageName, appName, combined); - } - if ("@walletCard".equals(template)) { - String wallet = walletPaymentMethod(combined); - return wallet.isEmpty() ? DEFAULT_PAYMENT_METHOD : wallet; - } - return safe(template).replace("${card}", safe(card)); - } - - private static String category(String merchant, String template) { - if ("@keyword".equals(template)) { - return categoryFor(merchant); - } - if ("@accountDebitKeyword".equals(template)) { - return accountDebitCategory(merchant); - } - return safe(template); - } - - private static String group(Matcher matcher, String original, String name) { - try { - int start = matcher.start(name); - int end = matcher.end(name); - if (start < 0 || end < 0) { - return ""; - } - return original.substring(start, end); - } catch (Exception ignored) { - return ""; - } - } - - private static String valueOrGroup(Matcher matcher, String original, String value, String group) { - if (value == null || value.isEmpty()) { - return safe(group(matcher, original, group)).toUpperCase(Locale.ROOT); - } - if (value.startsWith("$")) { - return safe(group(matcher, original, value.substring(1))).toUpperCase(Locale.ROOT); - } - return value; - } - - private static int flags(JSONArray json) { - int flags = 0; - if (json == null) { - return flags; - } - for (int i = 0; i < json.length(); i++) { - String flag = json.optString(i); - if ("caseInsensitive".equals(flag)) { - flags |= Pattern.CASE_INSENSITIVE; - } else if ("unicodeCase".equals(flag)) { - flags |= Pattern.UNICODE_CASE; - } else if ("multiline".equals(flag)) { - flags |= Pattern.MULTILINE; - } else if ("dotall".equals(flag)) { - flags |= Pattern.DOTALL; - } - } - return flags; - } - - private static List strings(JSONArray json, List fallback) { - if (json == null) { - return new ArrayList<>(fallback); - } - ArrayList result = new ArrayList<>(); - for (int i = 0; i < json.length(); i++) { - result.add(json.optString(i)); - } - return result; - } - - private static Set setFromJson(JSONArray json, Set fallback) { - if (json == null) { - return new HashSet<>(fallback); - } - return new HashSet<>(strings(json, Collections.emptyList())); - } - - private static Set lowerSet(List values) { - HashSet set = new HashSet<>(); - for (String value : values) { - set.add(safe(value).trim().toLowerCase(Locale.ROOT)); - } - return set; - } - - private static String stripJson(String fileName) { - return fileName.endsWith(".json") ? fileName.substring(0, fileName.length() - 5) : fileName; - } - - private static List list(String... values) { - ArrayList result = new ArrayList<>(); - Collections.addAll(result, values); - return result; - } - - private static Set set(String... values) { - HashSet result = new HashSet<>(); - Collections.addAll(result, values); - return result; - } - - private static final class DefaultJson { - static String global() { - return "{" - + "\"smsPackages\":[\"com.textra\",\"com.google.android.apps.messaging\",\"com.samsung.android.messaging\",\"com.android.mms\"]," - + "\"dropZeroAmount\":true," - + "\"rejectPhrases\":[" - + "\"waiting for your approval\",\"verify a payment\",\"verify your payment\",\"tap to start\"," - + "\"declined\",\"insufficient balance\",\"insufficient funds\",\"verified your card\"," - + "\"haven't been charged\",\"have not been charged\",\"card registration\",\"card verification\"]" - + "}"; - } - - static LinkedHashMap inputsByFile() { - LinkedHashMap inputs = new LinkedHashMap<>(); - inputs.put("revolut.json", "{" - + "\"id\":\"revolut\",\"displayName\":\"Revolut\",\"enabled\":true,\"priority\":10," - + "\"match\":{\"packages\":[\"com.revolut.revolut\",\"com.revolut.business\"]}," - + "\"transforms\":[\"normalizeAmount\",\"multiCurrency\"]," - + "\"rules\":[{\"name\":\"default\",\"type\":\"amountFallback\",\"merchantSource\":\"firstNonNoiseLine\"," - + "\"output\":{\"paymentMethod\":\"Credit Card\",\"category\":\"@keyword\"}}]}"); - inputs.put("google-wallet.json", "{" - + "\"id\":\"google-wallet\",\"displayName\":\"Google Wallet\",\"enabled\":true,\"priority\":20," - + "\"match\":{\"packages\":[\"com.google.android.apps.walletnfcrel\"]}," - + "\"rules\":[{\"name\":\"wallet-purchase\",\"type\":\"amountFallback\",\"merchantSource\":\"title\"," - + "\"output\":{\"paymentMethod\":\"@walletCard\",\"category\":\"@keyword\"}}]}"); - inputs.put("bank-of-cyprus.json", bankOfCyprusJson()); - inputs.put("eurobank.json", eurobankJson()); - inputs.put("alpha-bank.json", alphaBankJson()); - return inputs; - } - - private static String alphaBankJson() { - try { - JSONObject root = new JSONObject(); - root.put("id", "alpha-bank"); - root.put("displayName", "Alpha Bank"); - root.put("enabled", true); - root.put("priority", 50); - root.put("match", new JSONObject().put("senders", new JSONArray(list("alpha bank", "alphabank", "alpha alerts")))); - root.put("transforms", new JSONArray(list("normalizeAmount"))); - JSONArray rules = new JSONArray(); - rules.put(regexRule( - "card-authorised", - "YOUR\\s+CARD\\s+.+?\\*(?\\d{4})\\s+WAS\\s+AUTHORISED\\s+FOR\\s+AN\\s+INDICATIVE\\s+AMOUNT\\s+(?" + MONEY + ")\\s*(?EUR|USD|GBP)\\s+ON\\s+\\d{1,2}/\\d{1,2}/\\d{4}\\s+\\d{1,2}:\\d{2}\\s+AT\\s+(?.+?)\\s*$", - list("caseInsensitive"), - new JSONObject() - .put("paymentMethod", "Credit Card") - .put("category", "@keyword"))); - rules.put(regexRule( - "placed-transfer", - "YOU\\s+HAVE\\s+PLACED\\s+A\\s+TRANSFER\\s+TO\\s+(?\\S+)\\s+FOR\\s+(?" + MONEY + ")\\s*(?EUR|USD|GBP)\\s+WITH\\s+REF\\s+(?\\S+)\\s*$", - list("caseInsensitive"), - new JSONObject() - .put("paymentMethod", "Electronic Transfer") - .put("category", "Transfers"))); - rules.put(amountFallbackRule()); - root.put("rules", rules); - return root.toString(); - } catch (JSONException e) { - return "{}"; - } - } - - private static String bankOfCyprusJson() { - try { - JSONObject root = new JSONObject(); - root.put("id", "bank-of-cyprus"); - root.put("displayName", "Bank of Cyprus"); - root.put("enabled", true); - root.put("priority", 30); - root.put("match", new JSONObject().put("senders", new JSONArray(list("boc message")))); - root.put("transforms", new JSONArray(list("foldGreek", "normalizeAmount"))); - JSONArray rules = new JSONArray(); - rules.put(regexRule( - "card-use", - "\u0397\\s+\u039A\u0391\u03A1\u03A4\u0391\\s+\u03A3\u0391\u03A3\\s+[A-Z]+\\*?(?\\d{4})\\s+\u0395\u03A7\u0395\u0399\\s+\u03A7\u03A1\u0397\u03A3\u0399\u039C\u039F\u03A0\u039F\u0399\u0397\u0398\u0395\u0399\\s+\u03A3\u03A4\u039F\\s+(?[\\s\\S]+?)\\s+\u03A3\u03A4\u0399\u03A3\\s+\\d{1,2}/\\d{1,2}/\\d{4},\\s+\\d{1,2}:\\d{2}\\s+\u0393\u0399\u0391\\s+\u03A4\u039F\\s+\u0395\u039D\u0394\u0395\u0399\u039A\u03A4\u0399\u039A\u039F\\s+\u03A0\u039F\u03A3\u039F\\s+\\u20AC\\s*(?" + MONEY + ")\\.?", - list("caseInsensitive", "unicodeCase"), - new JSONObject() - .put("currency", "EUR") - .put("paymentMethod", "Credit Card") - .put("category", "@keyword"))); - rules.put(regexRule( - "account-debit", - "\u039F\\s+\u039B\u039F\u0393[\\s\\S]*?\\d{3,}[\\s\\S]*?\u03A7\u03A1\u0395\u03A9\u0398\u0397\u039A\u0395[\\s\\S]*?\u03A0\u039F\u03A3\u039F\\s+(?:\u03A4\u03A9\u039D\\s+)?(?EUR|USD|GBP)\\s*(?" + MONEY + ")[\\s\\S]*?\u03A0\u0395\u03A1\u0399\u0393\u03A1\u0391\u03A6\u0397\\s*:\\s*(?.+?)\\s*$", - list("caseInsensitive", "unicodeCase", "multiline"), - new JSONObject() - .put("paymentMethod", "Electronic Transfer") - .put("category", "@accountDebitKeyword"))); - rules.put(regexRule( - "incoming-credit", - "Ο\\s+ΛΟΓ[\\s\\S]*?\\d{3,}[\\s\\S]*?ΠΙΣΤΩΘΗΚΕ[\\s\\S]*?ΠΟΣΟ\\s+(?:ΤΩΝ\\s+)?(?EUR|USD|GBP)\\s*(?" + MONEY + ")[\\s\\S]*?ΠΕΡΙΓΡΑΦΗ\\s*:[\\s\\S]*?\\bBY\\s+(?[^>]+)(?:>[^>]+>(?.+?))?\\s*$", - list("caseInsensitive", "unicodeCase", "multiline"), - new JSONObject() - .put("type", "INCOME") - .put("paymentMethod", "Electronic Transfer") - .put("category", "Income"))); - rules.put(regexRule( - "incoming-credit-english", - "\\bcredited\\s+with\\s+the\\s+amount\\s+of\\s+(?EUR|USD|GBP)\\s*(?" + MONEY + ")[\\s\\S]*?\\bFrom:\\s*(?.+?)(?:\\s+Details:\\s*(?.+?))?\\s*$", - list("caseInsensitive"), - new JSONObject() - .put("type", "INCOME") - .put("paymentMethod", "Electronic Transfer") - .put("category", "Income"))); - rules.put(amountFallbackRule()); - root.put("rules", rules); - return root.toString(); - } catch (JSONException e) { - return "{}"; - } - } - - private static String eurobankJson() { - try { - JSONObject root = new JSONObject(); - root.put("id", "eurobank"); - root.put("displayName", "Eurobank Cyprus"); - root.put("enabled", true); - root.put("priority", 40); - root.put("match", new JSONObject().put("senders", new JSONArray(list("eurobank", "eurobankcy")))); - root.put("transforms", new JSONArray(list("foldGreek", "normalizeAmount"))); - JSONArray rules = new JSONArray(); - rules.put(regexRule( - "card-approved", - "\u0397\\s+\u039A\u0391\u03A1\u03A4\u0391\\s+\\*?(?\\d{4})\\s+\u0395\u0393\u039A\u03A1\u0399\u0398\u0397\u039A\u0395\\s+\u0393\u0399\u0391\\s+(?.+?)\\s+\\u20AC\\s*(?" + MONEY + ")\\s*(?:@\\d{1,2}:\\d{2})?\\s*$", - list("unicodeCase"), - new JSONObject() - .put("currency", "EUR") - .put("paymentMethod", "Credit Card") - .put("category", "@keyword"))); - rules.put(regexRule( - "incoming-credit", - "\u039B\u039F\u0393\u0391\u03A1\u0399\u0391\u03A3\u039C\u039F\u03A3[\\s\\S]*?\u03A0\u0399\u03A3\u03A4\u03A9\u0398\u0395\u0399[\\s\\S]*?\u03A0\u039F\u03A3\u039F\\s+\u03A4\u03A9\u039D\\s+(?" + MONEY + ")\\s*(?EUR|USD|GBP)", - list("caseInsensitive", "unicodeCase"), - new JSONObject() - .put("type", "INCOME") - .put("paymentMethod", "Electronic Transfer") - .put("category", "Income"))); - rules.put(amountFallbackRule()); - root.put("rules", rules); - return root.toString(); - } catch (JSONException e) { - return "{}"; - } - } - - private static JSONObject regexRule(String name, String pattern, List flags, JSONObject output) throws JSONException { - return new JSONObject() - .put("name", name) - .put("type", "regex") - .put("pattern", pattern) - .put("flags", new JSONArray(flags)) - .put("output", output); - } - - private static JSONObject amountFallbackRule() throws JSONException { - return new JSONObject() - .put("name", "unmatched-bank-amount") - .put("type", "amountFallback") - .put("merchantSource", "empty") - .put("noteSource", "body") - .put("output", new JSONObject() - .put("paymentMethod", "Electronic Transfer") - .put("category", "@keyword")); - } - } - - private static final class Amount { + static final class Amount { final String currency; final String value; @@ -1165,7 +432,7 @@ private static final class Amount { } } - private static final class MultiCurrencyAmount { + static final class MultiCurrencyAmount { final Amount account; final Amount original; diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/GlobalConfig.java b/android_app/app/src/main/java/dev/fanis/expensenotification/GlobalConfig.java new file mode 100644 index 0000000..ae48433 --- /dev/null +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/GlobalConfig.java @@ -0,0 +1,89 @@ +package dev.fanis.expensenotification; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Parser-wide settings: which SMS apps carry bank senders, phrases that mark a + * notification as not-a-charge, and whether zero amounts are dropped. Loaded from + * global.json (asset or user override); the built-in defaults are the safety net. + */ +final class GlobalConfig { + // Notifications that look payment-shaped but are not a completed charge: 3DS + // approval prompts (the duplicate that precedes a real "successful" message), + // declines, and card verification/registration. Matched as whole phrases so they + // don't catch a genuine payment whose merchant name happens to contain a word. + private static final String[] REJECT_PHRASES = { + "waiting for your approval", + "verify a payment", + "verify your payment", + "tap to start", + "declined", + "insufficient balance", + "insufficient funds", + "verified your card", + "haven't been charged", + "have not been charged", + "card registration", + "card verification", + }; + + private static final String[] SMS_PACKAGES = { + "com.textra", "com.google.android.apps.messaging", "com.samsung.android.messaging", "com.android.mms", + }; + + final Set smsPackages; + final List rejectPhrases; + final boolean dropZeroAmount; + + private GlobalConfig(Set smsPackages, List rejectPhrases, boolean dropZeroAmount) { + this.smsPackages = smsPackages; + this.rejectPhrases = rejectPhrases; + this.dropZeroAmount = dropZeroAmount; + } + + static GlobalConfig defaults() { + return new GlobalConfig( + new HashSet<>(Arrays.asList(SMS_PACKAGES)), + new ArrayList<>(Arrays.asList(REJECT_PHRASES)), + true); + } + + /** The defaults as JSON, for callers that need a global config document. */ + static String defaultJson() { + try { + GlobalConfig defaults = defaults(); + return new JSONObject() + .put("smsPackages", new JSONArray(new ArrayList<>(defaults.smsPackages))) + .put("dropZeroAmount", defaults.dropZeroAmount) + .put("rejectPhrases", new JSONArray(defaults.rejectPhrases)) + .toString(); + } catch (JSONException e) { + return "{}"; + } + } + + static GlobalConfig fromJson(JSONObject json) { + GlobalConfig defaults = defaults(); + return new GlobalConfig( + JsonValues.stringSet(json.optJSONArray("smsPackages"), defaults.smsPackages), + JsonValues.strings(json.optJSONArray("rejectPhrases"), defaults.rejectPhrases), + json.optBoolean("dropZeroAmount", true)); + } + + GlobalConfig withSmsPackages(Set extraSmsPackages) { + if (extraSmsPackages == null || extraSmsPackages.isEmpty()) { + return this; + } + HashSet merged = new HashSet<>(smsPackages); + merged.addAll(extraSmsPackages); + return new GlobalConfig(merged, rejectPhrases, dropZeroAmount); + } +} diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/InputRule.java b/android_app/app/src/main/java/dev/fanis/expensenotification/InputRule.java new file mode 100644 index 0000000..c8ab174 --- /dev/null +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/InputRule.java @@ -0,0 +1,215 @@ +package dev.fanis.expensenotification; + +import org.json.JSONObject; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/** + * One parsing rule inside an input source. Regex rules compile their pattern once, + * at construction (folded to Latin lookalikes when the source uses foldGreek), so + * matching a notification never pays a Pattern.compile. + */ +final class InputRule { + final String name; + final String type; + // Compiled at construction; null for non-regex rules and for patterns that do + // not compile (a bad user pattern makes the rule inert instead of crashing the + // notification listener at parse time). + private final Pattern pattern; + private final boolean foldGreek; + private final boolean multiCurrency; + private final JSONObject output; + private final String merchantSource; + private final String noteSource; + + private InputRule(String name, String type, Pattern pattern, boolean foldGreek, boolean multiCurrency, + JSONObject output, String merchantSource, String noteSource) { + this.name = name; + this.type = type; + this.pattern = pattern; + this.foldGreek = foldGreek; + this.multiCurrency = multiCurrency; + this.output = output == null ? new JSONObject() : output; + this.merchantSource = merchantSource; + this.noteSource = noteSource; + } + + static InputRule fromJson(JSONObject json, List sourceTransforms) { + List transforms = JsonValues.strings(json.optJSONArray("transforms"), Collections.emptyList()); + boolean foldGreek = sourceTransforms.contains("foldGreek") || transforms.contains("foldGreek"); + boolean multiCurrency = sourceTransforms.contains("multiCurrency") || transforms.contains("multiCurrency"); + String type = json.optString("type", "regex"); + String rawPattern = json.optString("pattern", ""); + Pattern pattern = null; + if ("regex".equals(type) && !rawPattern.isEmpty()) { + try { + pattern = Pattern.compile( + foldGreek ? ExpenseParser.foldAmbiguousGreek(rawPattern) : rawPattern, + JsonValues.patternFlags(json.optJSONArray("flags"))); + } catch (PatternSyntaxException ignored) { + // Leave the rule inert; ConfigValidator reports the broken pattern. + } + } + return new InputRule( + json.optString("name", ""), + type, + pattern, + foldGreek, + multiCurrency, + json.optJSONObject("output"), + json.optString("merchantSource", ""), + json.optString("noteSource", "")); + } + + Candidate parse(GlobalConfig global, String packageName, String appName, String key, long postedAt, + String title, String body, NotificationText text) { + if ("amountFallback".equals(type)) { + return amountFallback(global, packageName, appName, key, postedAt, title, body, text.combined); + } + if (pattern == null) { + return null; + } + return regexCandidate(global, packageName, appName, key, postedAt, title, body, text); + } + + private Candidate amountFallback(GlobalConfig global, String packageName, String appName, String key, long postedAt, + String title, String body, String combined) { + ExpenseParser.MultiCurrencyAmount multi = multiCurrency ? ExpenseParser.extractMultiCurrencyAmount(combined) : null; + ExpenseParser.Amount amount = multi != null ? multi.account : ExpenseParser.extractAmount(combined); + if (amount == null) { + return null; + } + if (global.dropZeroAmount && ExpenseParser.isZero(amount.value)) { + return null; + } + Candidate candidate = baseCandidate(packageName, appName, key, postedAt, title, body); + candidate.amount = amount.value; + candidate.currency = amount.currency; + if (multi != null) { + candidate.originalAmount = multi.original.value; + candidate.originalCurrency = multi.original.currency; + } + candidate.merchant = merchantFromSource(title, body, amount); + candidate.note = "body".equals(noteSource) ? ExpenseParser.safe(body) : ""; + candidate.suggestedPaymentMethod = paymentMethod(combined, packageName, appName, + output.optString("paymentMethod", "@packageDefault"), ""); + candidate.suggestedCategory = category(candidate.merchant, output.optString("category", "@keyword")); + candidate.status = Candidate.STATUS_NEW; + return ExpenseParser.isInteresting(candidate) ? candidate : null; + } + + private Candidate regexCandidate(GlobalConfig global, String packageName, String appName, String key, long postedAt, + String title, String body, NotificationText text) { + // Folding is 1:1 and length-preserving, so group offsets from the folded + // target remain valid against the original combined text. + Matcher matcher = pattern.matcher(foldGreek ? text.folded() : text.combined); + if (!matcher.find()) { + return null; + } + String combined = text.combined; + String amount = ExpenseParser.normalizeAmount(group(matcher, combined, "amount")); + if (amount.isEmpty()) { + return null; + } + if (global.dropZeroAmount && ExpenseParser.isZero(amount)) { + return null; + } + Candidate candidate = baseCandidate(packageName, appName, key, postedAt, title, body); + candidate.amount = amount; + candidate.currency = valueOrGroup(matcher, combined, output.optString("currency", ""), "currency"); + String merchant = group(matcher, combined, "merchant"); + if (merchant.isEmpty()) { + merchant = merchantFromSource(title, body, new ExpenseParser.Amount(candidate.currency, candidate.amount)); + } + candidate.merchant = ExpenseParser.cleanupMerchant(merchant); + candidate.note = group(matcher, combined, "note"); + if (candidate.note.isEmpty() && "body".equals(noteSource)) { + candidate.note = ExpenseParser.safe(body); + } + String card = group(matcher, combined, "card"); + candidate.suggestedPaymentMethod = paymentMethod(combined, packageName, appName, + output.optString("paymentMethod", "@packageDefault"), card); + candidate.suggestedCategory = category(candidate.merchant, output.optString("category", "@keyword")); + candidate.transactionType = output.optString("type", Candidate.TYPE_EXPENSE); + if (candidate.isIncome() && candidate.suggestedCategory.isEmpty()) { + candidate.suggestedCategory = "Income"; + } + candidate.status = Candidate.STATUS_NEW; + return candidate; + } + + private static Candidate baseCandidate(String packageName, String appName, String key, long postedAt, + String title, String body) { + Candidate candidate = new Candidate(); + candidate.notificationKey = key; + candidate.packageName = packageName; + candidate.appName = appName; + candidate.title = ExpenseParser.safe(title); + candidate.text = ExpenseParser.safe(body); + candidate.postedAt = ExpenseParser.transactionTime(title, body, postedAt); + candidate.originalAmount = ""; + candidate.originalCurrency = ""; + candidate.note = ""; + candidate.transactionType = Candidate.TYPE_EXPENSE; + return candidate; + } + + private String merchantFromSource(String title, String body, ExpenseParser.Amount amount) { + if ("empty".equals(merchantSource)) { + return ""; + } + if ("title".equals(merchantSource)) { + return ExpenseParser.cleanupMerchant(title); + } + return ExpenseParser.guessMerchant(title, body, amount); + } + + private static String paymentMethod(String combined, String packageName, String appName, String template, String card) { + if ("@packageDefault".equals(template)) { + return ExpenseParser.paymentMethodFor(packageName, appName, combined); + } + if ("@walletCard".equals(template)) { + String wallet = ExpenseParser.walletPaymentMethod(combined); + return wallet.isEmpty() ? ExpenseParser.DEFAULT_PAYMENT_METHOD : wallet; + } + return ExpenseParser.safe(template).replace("${card}", ExpenseParser.safe(card)); + } + + private static String category(String merchant, String template) { + if ("@keyword".equals(template)) { + return ExpenseParser.categoryFor(merchant); + } + if ("@accountDebitKeyword".equals(template)) { + return ExpenseParser.accountDebitCategory(merchant); + } + return ExpenseParser.safe(template); + } + + private static String group(Matcher matcher, String original, String name) { + try { + int start = matcher.start(name); + int end = matcher.end(name); + if (start < 0 || end < 0) { + return ""; + } + return original.substring(start, end); + } catch (Exception ignored) { + return ""; + } + } + + private static String valueOrGroup(Matcher matcher, String original, String value, String group) { + if (value == null || value.isEmpty()) { + return group(matcher, original, group).toUpperCase(Locale.ROOT); + } + if (value.startsWith("$")) { + return group(matcher, original, value.substring(1)).toUpperCase(Locale.ROOT); + } + return value; + } +} diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/InputSource.java b/android_app/app/src/main/java/dev/fanis/expensenotification/InputSource.java new file mode 100644 index 0000000..30cd4e3 --- /dev/null +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/InputSource.java @@ -0,0 +1,71 @@ +package dev.fanis.expensenotification; + +import org.json.JSONArray; +import org.json.JSONObject; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +/** One input config file: how to recognize its notifications and the rules to parse them. */ +final class InputSource { + final String id; + final String displayName; + final boolean enabled; + final int priority; + final Set packages; + final Set senders; + final List rules; + + private InputSource(String id, String displayName, boolean enabled, int priority, Set packages, + Set senders, List rules) { + this.id = id; + this.displayName = displayName; + this.enabled = enabled; + this.priority = priority; + this.packages = packages; + this.senders = senders; + this.rules = rules; + } + + static InputSource fromJson(String fileName, JSONObject json) { + JSONObject match = json.optJSONObject("match"); + List transforms = JsonValues.strings(json.optJSONArray("transforms"), Collections.emptyList()); + List rules = new ArrayList<>(); + JSONArray rawRules = json.optJSONArray("rules"); + if (rawRules != null) { + for (int i = 0; i < rawRules.length(); i++) { + JSONObject rawRule = rawRules.optJSONObject(i); + if (rawRule != null) { + rules.add(InputRule.fromJson(rawRule, transforms)); + } + } + } + return new InputSource( + json.optString("id", stripJson(fileName)), + json.optString("displayName", stripJson(fileName)), + json.optBoolean("enabled", true), + json.optInt("priority", 100), + JsonValues.stringSet(match == null ? null : match.optJSONArray("packages"), Collections.emptySet()), + JsonValues.lowerSet(JsonValues.strings(match == null ? null : match.optJSONArray("senders"), Collections.emptyList())), + rules); + } + + boolean matches(String packageName, String title, Set smsPackages) { + String safePackage = packageName == null ? "" : packageName; + if (packages.contains(safePackage)) { + return true; + } + if (!senders.isEmpty() && smsPackages.contains(safePackage)) { + String lowerTitle = (title == null ? "" : title).trim().toLowerCase(Locale.ROOT); + return senders.contains(lowerTitle); + } + return packages.isEmpty() && senders.isEmpty(); + } + + private static String stripJson(String fileName) { + return fileName.endsWith(".json") ? fileName.substring(0, fileName.length() - 5) : fileName; + } +} diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/IoUtil.java b/android_app/app/src/main/java/dev/fanis/expensenotification/IoUtil.java new file mode 100644 index 0000000..d3a6e38 --- /dev/null +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/IoUtil.java @@ -0,0 +1,38 @@ +package dev.fanis.expensenotification; + +import android.content.res.AssetManager; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +/** Small-file read helpers shared by the config loaders. */ +final class IoUtil { + private IoUtil() { + } + + static String readAsset(AssetManager assets, String name) throws IOException { + try (InputStream in = assets.open(name)) { + return readAll(in); + } + } + + static String readFile(File file) throws IOException { + try (InputStream in = new FileInputStream(file)) { + return readAll(in); + } + } + + static String readAll(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int read; + while ((read = in.read(buffer)) != -1) { + out.write(buffer, 0, read); + } + return out.toString(StandardCharsets.UTF_8.name()); + } +} diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/JsonValues.java b/android_app/app/src/main/java/dev/fanis/expensenotification/JsonValues.java new file mode 100644 index 0000000..b762586 --- /dev/null +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/JsonValues.java @@ -0,0 +1,64 @@ +package dev.fanis.expensenotification; + +import org.json.JSONArray; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Pattern; + +/** JSON array/scalar coercion helpers shared by the config parser and validator. */ +final class JsonValues { + private JsonValues() { + } + + static List strings(JSONArray json, List fallback) { + if (json == null) { + return new ArrayList<>(fallback); + } + ArrayList result = new ArrayList<>(); + for (int i = 0; i < json.length(); i++) { + result.add(json.optString(i)); + } + return result; + } + + static Set stringSet(JSONArray json, Set fallback) { + if (json == null) { + return new HashSet<>(fallback); + } + return new HashSet<>(strings(json, Collections.emptyList())); + } + + static Set lowerSet(List values) { + HashSet set = new HashSet<>(); + for (String value : values) { + set.add(value == null ? "" : value.trim().toLowerCase(Locale.ROOT)); + } + return set; + } + + /** Maps config flag names to java.util.regex compile flags. */ + static int patternFlags(JSONArray json) { + int flags = 0; + if (json == null) { + return flags; + } + for (int i = 0; i < json.length(); i++) { + String flag = json.optString(i); + if ("caseInsensitive".equals(flag)) { + flags |= Pattern.CASE_INSENSITIVE; + } else if ("unicodeCase".equals(flag)) { + flags |= Pattern.UNICODE_CASE; + } else if ("multiline".equals(flag)) { + flags |= Pattern.MULTILINE; + } else if ("dotall".equals(flag)) { + flags |= Pattern.DOTALL; + } + } + return flags; + } +} diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/MainActivity.java b/android_app/app/src/main/java/dev/fanis/expensenotification/MainActivity.java index f205f18..56c4103 100644 --- a/android_app/app/src/main/java/dev/fanis/expensenotification/MainActivity.java +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/MainActivity.java @@ -1,6 +1,7 @@ package dev.fanis.expensenotification; import android.app.AlertDialog; +import android.content.ActivityNotFoundException; import android.content.Intent; import android.graphics.Typeface; import android.os.Bundle; @@ -10,27 +11,30 @@ import android.widget.LinearLayout; import android.widget.ScrollView; import android.widget.TextView; +import android.widget.Toast; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; import java.util.Locale; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; public class MainActivity extends BaseActivity { - private static final String APP_PACKAGE = "dev.fanis.expensenotification"; - private static final String NOTIFICATION_LISTENER = APP_PACKAGE + "/dev.fanis.expensenotification.ExpenseNotificationListener"; - private static final String ACCESSIBILITY_SERVICE = APP_PACKAGE + "/dev.fanis.expensenotification.ExpenseEntryAccessibilityService"; private CandidateDb db; private LinearLayout setupActions; private LinearLayout list; private TextView status; + // Candidate loading runs the reparse-on-config-change pass, so keep it off the + // UI thread; single-threaded so a stale refresh can't overtake a newer one. + private final ExecutorService loader = Executors.newSingleThreadExecutor(); @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); - db = new CandidateDb(this); + db = CandidateDb.getInstance(this); // Keep the notification listener revived if the system unbinds or kills it. ListenerWatchdogJob.schedule(this); setContentView(buildUi()); @@ -45,6 +49,12 @@ protected void onResume() { } } + @Override + protected void onDestroy() { + super.onDestroy(); + loader.shutdown(); + } + @Override protected boolean showSettingsInHeader() { return true; @@ -73,8 +83,18 @@ private View buildUi() { private void refresh() { renderSetupActions(); + loader.execute(() -> { + List candidates = db.listAll(); + runOnUiThread(() -> { + if (!isFinishing() && !isDestroyed()) { + renderList(candidates); + } + }); + }); + } + + private void renderList(List candidates) { list.removeAllViews(); - List candidates = db.listAll(); status.setText(candidates.size() + " captured notification(s). New payment notifications are captured automatically."); if (candidates.isEmpty()) { LinearLayout emptyCard = new LinearLayout(this); @@ -154,8 +174,10 @@ private View card(Candidate candidate) { " - " + emptyDash(candidate.suggestedCategory))); card.addView(bodyText("From " + emptyDash(candidate.appName))); card.addView(bodyText(DateFormat.getDateTimeInstance().format(new Date(candidate.postedAt)))); - if ("SKIPPED".equals(candidate.status) || "PROCESSED".equals(candidate.status)) { - TextView statusLabel = bodyText(statusLabel(candidate.status)); + boolean skipped = Candidate.STATUS_SKIPPED.equals(candidate.status); + boolean processed = Candidate.STATUS_PROCESSED.equals(candidate.status); + if (skipped || processed) { + TextView statusLabel = bodyText(processed ? "Processed" : "Skipped"); statusLabel.setTextSize(13); statusLabel.setPadding(0, dp(6), 0, 0); card.addView(statusLabel); @@ -166,17 +188,17 @@ private View card(Candidate candidate) { LinearLayout actions = new LinearLayout(this); actions.setOrientation(LinearLayout.HORIZONTAL); - if ("SKIPPED".equals(candidate.status)) { + if (skipped) { Button unskip = secondaryButton("Unskip"); unskip.setOnClickListener(v -> { - db.mark(candidate.id, "NEW"); + db.mark(candidate.id, Candidate.STATUS_NEW); refresh(); }); addCardAction(actions, unskip); - } else if ("PROCESSED".equals(candidate.status)) { + } else if (processed) { Button reopen = secondaryButton("Mark new"); reopen.setOnClickListener(v -> { - db.mark(candidate.id, "NEW"); + db.mark(candidate.id, Candidate.STATUS_NEW); refresh(); }); addCardAction(actions, reopen); @@ -186,7 +208,7 @@ private View card(Candidate candidate) { addCardAction(actions, fill); Button skip = secondaryButton("Skip"); skip.setOnClickListener(v -> { - db.mark(candidate.id, "SKIPPED"); + db.mark(candidate.id, Candidate.STATUS_SKIPPED); refresh(); }); addCardAction(actions, skip); @@ -195,16 +217,6 @@ private View card(Candidate candidate) { return card; } - private String statusLabel(String status) { - if ("PROCESSED".equals(status)) { - return "Processed"; - } - if ("SKIPPED".equals(status)) { - return "Skipped"; - } - return status == null ? "" : status; - } - private void addCardAction(LinearLayout actions, Button button) { LinearLayout.LayoutParams params = new LinearLayout.LayoutParams( 0, @@ -252,8 +264,10 @@ private void fillExpenseManagerWithAmount(Candidate candidate, String amount) { // from the SMS, or the notification post time as a fallback), not now(): a // candidate may sit in the queue for days before the user fills it. String date = expenseDate(candidate.postedAt, profile.dateFormat); - fillExpenseManager(profile, amount, candidate.merchant, candidate.suggestedPaymentMethod, description, category, date); - db.mark(candidate.id, "PROCESSED"); + if (!fillExpenseManager(profile, amount, candidate.merchant, candidate.suggestedPaymentMethod, description, category, date)) { + return; + } + db.mark(candidate.id, Candidate.STATUS_PROCESSED); refresh(); } @@ -269,7 +283,8 @@ private static String expenseDate(long postedAt, String dateFormat) { } } - private void fillExpenseManager(OutputProfile profile, String amount, String merchant, String paymentMethod, String description, String category, String date) { + /** Launches the output app's add-transaction form. Returns false when it can't be opened. */ + private boolean fillExpenseManager(OutputProfile profile, String amount, String merchant, String paymentMethod, String description, String category, String date) { // Prefill the learned payee for this merchant when we have one; otherwise the // raw merchant. We keep the raw merchant separately so the accessibility // service can learn merchant -> payee from whatever the user finally selects. @@ -283,10 +298,9 @@ private void fillExpenseManager(OutputProfile profile, String amount, String mer .putString("amount", amount) .putString("merchant", merchant == null ? "" : merchant) .putString("payee", prefillPayee) - .putString("payment_method", paymentMethod) .putString("description", description) - .putString("pending_payee", "") .putString("state", "PENDING") + .putLong("state_at", System.currentTimeMillis()) .apply(); Intent intent = new Intent(); intent.setClassName(profile.packageName, profile.activity); @@ -311,7 +325,16 @@ private void fillExpenseManager(OutputProfile profile, String amount, String mer if (date != null && !date.isEmpty()) { intent.putExtra(profile.dateExtra(), date); } - startActivity(intent); + try { + startActivity(intent); + return true; + } catch (ActivityNotFoundException | SecurityException e) { + // The output app is not installed (or its add-transaction screen moved); + // don't mark the candidate processed for a fill that never happened. + getSharedPreferences("automation", MODE_PRIVATE).edit().putString("state", "").apply(); + Toast.makeText(this, "Could not open " + profile.displayName + ". Is it installed?", Toast.LENGTH_LONG).show(); + return false; + } } private String displayAmountLine(Candidate candidate) { @@ -324,29 +347,4 @@ private String displayAmountLine(Candidate candidate) { } return candidate.amountLine() + " (was " + candidate.originalAmountLine() + ")"; } - - private boolean isNotificationListenerEnabled() { - String enabled = Settings.Secure.getString(getContentResolver(), "enabled_notification_listeners"); - return containsComponent(enabled, NOTIFICATION_LISTENER); - } - - private boolean isAccessibilityServiceEnabled() { - String enabled = Settings.Secure.getString(getContentResolver(), "enabled_accessibility_services"); - return "1".equals(Settings.Secure.getString(getContentResolver(), "accessibility_enabled")) && - containsComponent(enabled, ACCESSIBILITY_SERVICE); - } - - private static boolean containsComponent(String enabled, String flattenedComponent) { - if (enabled == null || enabled.isEmpty()) { - return false; - } - String shortComponent = flattenedComponent.replace("/" + APP_PACKAGE + ".", "/."); - for (String component : enabled.split(":")) { - if (component.equals(flattenedComponent) || component.equals(shortComponent)) { - return true; - } - } - return false; - } - } diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/NotificationText.java b/android_app/app/src/main/java/dev/fanis/expensenotification/NotificationText.java new file mode 100644 index 0000000..005682b --- /dev/null +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/NotificationText.java @@ -0,0 +1,21 @@ +package dev.fanis.expensenotification; + +/** + * The combined title+body of one notification, with the Greek-folded variant + * computed at most once no matter how many fold-aware rules inspect it. + */ +final class NotificationText { + final String combined; + private String folded; + + NotificationText(String combined) { + this.combined = combined; + } + + String folded() { + if (folded == null) { + folded = ExpenseParser.foldAmbiguousGreek(combined); + } + return folded; + } +} diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/OutputProfile.java b/android_app/app/src/main/java/dev/fanis/expensenotification/OutputProfile.java index 0fbcc89..89e52c3 100644 --- a/android_app/app/src/main/java/dev/fanis/expensenotification/OutputProfile.java +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/OutputProfile.java @@ -7,12 +7,8 @@ import org.json.JSONException; import org.json.JSONObject; -import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -101,21 +97,23 @@ static OutputProfile defaults() { } private static OutputProfile load(Context context) throws IOException, JSONException { + // The bundled assets are the single source of defaults; user files override + // assets of the same name and hidden bundled outputs are excluded, so + // deleting one from the Config screen really removes it. LinkedHashMap outputs = new LinkedHashMap<>(); - outputs.put("expense-manager.json", defaultJson()); AssetManager assets = context.getAssets(); try { String[] names = assets.list("outputs"); if (names != null) { ArrayList sorted = new ArrayList<>(); - Collections.addAll(sorted, names); - Collections.sort(sorted); - for (String name : sorted) { - if (name.endsWith(".json") && !ConfigHides.isHidden(context, "outputs", name)) { - outputs.put(name, readAsset(assets, "outputs/" + name)); - } + Collections.addAll(sorted, names); + Collections.sort(sorted); + for (String name : sorted) { + if (name.endsWith(".json") && !ConfigHides.isHidden(context, "outputs", name)) { + outputs.put(name, IoUtil.readAsset(assets, "outputs/" + name)); } + } } } catch (IOException ignored) { } @@ -128,7 +126,7 @@ private static OutputProfile load(Context context) throws IOException, JSONExcep Collections.sort(sorted, (a, b) -> a.getName().compareTo(b.getName())); for (File file : sorted) { if (file.isFile() && file.getName().endsWith(".json")) { - outputs.put(file.getName(), readFile(file)); + outputs.put(file.getName(), IoUtil.readFile(file)); } } } @@ -247,42 +245,4 @@ String summary() { + "Accessibility save ids: " + (saveIds.isEmpty() ? "-" : saveIdsPrefValue()); } - private static String readAsset(AssetManager assets, String name) throws IOException { - try (InputStream in = assets.open(name)) { - return readAll(in); - } - } - - private static String readFile(File file) throws IOException { - try (InputStream in = new FileInputStream(file)) { - return readAll(in); - } - } - - private static String readAll(InputStream in) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - byte[] buffer = new byte[4096]; - int read; - while ((read = in.read(buffer)) != -1) { - out.write(buffer, 0, read); - } - return out.toString(StandardCharsets.UTF_8.name()); - } - - private static String defaultJson() { - return "{" - + "\"id\":\"expense-manager\"," - + "\"displayName\":\"Bishinews Expense Manager\"," - + "\"package\":\"com.expensemanager.pro\"," - + "\"activity\":\"com.expensemanager.ExpenseNewTransaction\"," - + "\"constantExtras\":{\"fromWhere\":\"widgetAdd\"}," - + "\"fieldMap\":{\"amount\":\"amount\",\"payee\":\"payee\",\"paymentMethod\":\"paymentMethod\"," - + "\"category\":\"category\",\"description\":\"description\",\"date\":\"date\"}," - + "\"dateFormat\":\"yyyy-MM-dd\"," - + "\"accessibility\":{\"amountId\":\"com.expensemanager.pro:id/expenseAmountInput\"," - + "\"payeeId\":\"com.expensemanager.pro:id/payee\"," - + "\"descriptionId\":\"com.expensemanager.pro:id/expenseDescriptionInput\"," - + "\"saveIds\":[\"com.expensemanager.pro:id/expenseSave\",\"com.expensemanager.pro:id/expenseSaveNew\"]}" - + "}"; - } } diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/PayeeAliases.java b/android_app/app/src/main/java/dev/fanis/expensenotification/PayeeAliases.java index 36b10ea..206aa47 100644 --- a/android_app/app/src/main/java/dev/fanis/expensenotification/PayeeAliases.java +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/PayeeAliases.java @@ -96,10 +96,6 @@ static Map all(Context context) { return result; } - static List keys(Context context) { - return new ArrayList<>(all(context).keySet()); - } - static void remove(Context context, String key) { store(context).edit().remove(key).apply(); } @@ -110,10 +106,6 @@ static int count(Context context) { // ---- Blacklist: merchants that must never be auto-learned or auto-mapped. ---- - static boolean isBlacklisted(Context context, String merchant) { - return isBlacklistedKey(context, normalize(merchant)); - } - private static boolean isBlacklistedKey(Context context, String key) { return !key.isEmpty() && blacklistStore(context).contains(key); } diff --git a/android_app/app/src/main/java/dev/fanis/expensenotification/SettingsActivity.java b/android_app/app/src/main/java/dev/fanis/expensenotification/SettingsActivity.java index 0e843e5..ee56956 100644 --- a/android_app/app/src/main/java/dev/fanis/expensenotification/SettingsActivity.java +++ b/android_app/app/src/main/java/dev/fanis/expensenotification/SettingsActivity.java @@ -1,6 +1,7 @@ package dev.fanis.expensenotification; import android.app.AlertDialog; +import android.content.ActivityNotFoundException; import android.content.Intent; import android.content.pm.PackageInfo; import android.content.pm.PackageManager; @@ -24,7 +25,7 @@ public class SettingsActivity extends BaseActivity { @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); - db = new CandidateDb(this); + db = CandidateDb.getInstance(this); setContentView(buildUi()); } @@ -88,15 +89,24 @@ private View buildUi() { root.addView(batteryButton); Button clear = dangerButton("Clear local queue"); - clear.setOnClickListener(v -> { - int deleted = db.deleteAll(); - Toast.makeText(this, "Deleted " + deleted + " item(s).", Toast.LENGTH_SHORT).show(); - }); + clear.setOnClickListener(v -> confirmClearQueue()); root.addView(clear); return scroll; } + private void confirmClearQueue() { + new AlertDialog.Builder(this) + .setTitle("Clear local queue?") + .setMessage("This permanently deletes every captured notification, including ones you have not reviewed yet.") + .setPositiveButton("Delete all", (dialog, which) -> { + int deleted = db.deleteAll(); + Toast.makeText(this, "Deleted " + deleted + " item(s).", Toast.LENGTH_SHORT).show(); + }) + .setNegativeButton("Cancel", null) + .show(); + } + private void scanNow() { int saved = ExpenseNotificationListener.scanActive(this); if (saved == -1) { @@ -108,9 +118,14 @@ private void scanNow() { } private void openExpenseManager() { + OutputProfile profile = OutputProfile.active(this); Intent intent = new Intent(); - intent.setClassName("com.expensemanager.pro", "com.expensemanager.ExpenseNewTransaction"); - startActivity(intent); + intent.setClassName(profile.packageName, profile.activity); + try { + startActivity(intent); + } catch (ActivityNotFoundException | SecurityException e) { + Toast.makeText(this, "Could not open " + profile.displayName + ". Is it installed?", Toast.LENGTH_LONG).show(); + } } private String batteryLabel() { @@ -178,21 +193,6 @@ private String notificationSummary() { return "Notification access " + enabledWord(isNotificationListenerEnabled()); } - private boolean isNotificationListenerEnabled() { - String enabled = Settings.Secure.getString(getContentResolver(), "enabled_notification_listeners"); - String component = getPackageName() + "/" + getPackageName() + ".ExpenseNotificationListener"; - String shortComponent = getPackageName() + "/.ExpenseNotificationListener"; - return enabled != null && (enabled.contains(component) || enabled.contains(shortComponent)); - } - - private boolean isAccessibilityServiceEnabled() { - String enabled = Settings.Secure.getString(getContentResolver(), "enabled_accessibility_services"); - String component = getPackageName() + "/" + getPackageName() + ".ExpenseEntryAccessibilityService"; - String shortComponent = getPackageName() + "/.ExpenseEntryAccessibilityService"; - return "1".equals(Settings.Secure.getString(getContentResolver(), "accessibility_enabled")) && - enabled != null && (enabled.contains(component) || enabled.contains(shortComponent)); - } - private static String enabledWord(boolean enabled) { return enabled ? "enabled" : "missing"; } diff --git a/android_app/app/src/test/java/dev/fanis/expensenotification/CandidateDbRobolectricTest.java b/android_app/app/src/test/java/dev/fanis/expensenotification/CandidateDbRobolectricTest.java new file mode 100644 index 0000000..f092cc7 --- /dev/null +++ b/android_app/app/src/test/java/dev/fanis/expensenotification/CandidateDbRobolectricTest.java @@ -0,0 +1,92 @@ +package dev.fanis.expensenotification; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import android.content.Context; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricTestRunner; +import org.robolectric.RuntimeEnvironment; + +import java.util.List; + +@RunWith(RobolectricTestRunner.class) +public class CandidateDbRobolectricTest { + + private Context context; + private CandidateDb db; + + @Before + public void setUp() { + CandidateDb.resetInstanceForTesting(); + context = RuntimeEnvironment.getApplication(); + db = CandidateDb.getInstance(context); + } + + @After + public void tearDown() { + CandidateDb.resetInstanceForTesting(); + } + + private static Candidate candidate(String key, long postedAt) { + Candidate candidate = new Candidate(); + candidate.notificationKey = key; + candidate.packageName = "com.revolut.revolut"; + candidate.appName = "Revolut"; + candidate.title = "SAMPLE UTILITY"; + candidate.text = "You spent €33.95\nEUR balance: €543.21"; + candidate.merchant = "SAMPLE UTILITY"; + candidate.amount = "33.95"; + candidate.currency = "EUR"; + candidate.suggestedCategory = ""; + candidate.suggestedPaymentMethod = "Credit Card"; + candidate.postedAt = postedAt; + candidate.status = Candidate.STATUS_NEW; + return candidate; + } + + @Test + public void duplicateNotificationKeyIsIgnored() { + assertNotEquals(-1, db.insertIfNew(candidate("key-1", 1000L))); + assertEquals(-1, db.insertIfNew(candidate("key-1", 1000L))); + assertEquals(1, db.listAll().size()); + } + + @Test + public void reparsesStoredSmsOnceAfterConfigRevisionChanges() { + Candidate stale = candidate("key-1", 1000L); + stale.merchant = "STALE MERCHANT"; + stale.amount = "999.99"; + assertNotEquals(-1, db.insertIfNew(stale)); + + // Same revision: the stored fields are returned untouched. + assertEquals("STALE MERCHANT", db.listAll().get(0).merchant); + + // After a config change the stored SMS is re-parsed with the current parser + // and the corrected fields are written back. + ConfigRevision.bump(context); + Candidate refreshed = db.listAll().get(0); + assertEquals("SAMPLE UTILITY", refreshed.merchant); + assertEquals("33.95", refreshed.amount); + assertEquals(Candidate.STATUS_NEW, refreshed.status); + } + + @Test + public void prunesOldSettledCandidatesButKeepsNewOnes() { + long old = System.currentTimeMillis() - CandidateDb.SETTLED_RETENTION_MS - 1000L; + long settledId = db.insertIfNew(candidate("old-processed", old)); + assertTrue(settledId != -1); + assertNotEquals(-1, db.insertIfNew(candidate("old-new", old + 1))); + db.mark(settledId, Candidate.STATUS_PROCESSED); + + assertEquals(1, db.pruneSettled()); + List remaining = db.listAll(); + assertEquals(1, remaining.size()); + assertEquals("old-new", remaining.get(0).notificationKey); + } +} diff --git a/android_app/app/src/test/java/dev/fanis/expensenotification/ConfigNamesTest.java b/android_app/app/src/test/java/dev/fanis/expensenotification/ConfigNamesTest.java new file mode 100644 index 0000000..92f8275 --- /dev/null +++ b/android_app/app/src/test/java/dev/fanis/expensenotification/ConfigNamesTest.java @@ -0,0 +1,39 @@ +package dev.fanis.expensenotification; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class ConfigNamesTest { + + @Test + public void cleansUnsafeFileNames() { + assertEquals("my-bank.json", ConfigNames.cleanJsonName("my-bank")); + assertEquals("my-bank.json", ConfigNames.cleanJsonName(" my-bank.json ")); + assertEquals("a-b.json", ConfigNames.cleanJsonName("a/b")); + assertEquals("config.json", ConfigNames.cleanJsonName(null)); + assertEquals("config.json", ConfigNames.cleanJsonName(" ")); + } + + @Test + public void decodesJsonLevelUnicodeEscapes() { + assertEquals("Η ΚΑΡΤΑ", ConfigNames.decodeUnicodeEscapes("\\u0397 \\u039A\\u0391\\u03A1\\u03A4\\u0391")); + assertEquals("plain text", ConfigNames.decodeUnicodeEscapes("plain text")); + } + + @Test + public void leavesEscapedBackslashUnicodeSequencesAlone() { + // \\u20AC inside a JSON string is a literal backslash + "u20AC" (e.g. a + // regex-level unicode escape); decoding it would corrupt the pattern. + assertEquals("\\\\u20AC", ConfigNames.decodeUnicodeEscapes("\\\\u20AC")); + // Three backslashes: an escaped backslash followed by a real unicode escape. + assertEquals("\\\\€", ConfigNames.decodeUnicodeEscapes("\\\\\\u20AC")); + } + + @Test + public void leavesMalformedEscapesAlone() { + assertEquals("\\uZZZZ", ConfigNames.decodeUnicodeEscapes("\\uZZZZ")); + assertEquals("\\u12", ConfigNames.decodeUnicodeEscapes("\\u12")); + assertEquals("trailing\\", ConfigNames.decodeUnicodeEscapes("trailing\\")); + } +} diff --git a/android_app/app/src/test/java/dev/fanis/expensenotification/ConfigRuntimeRobolectricTest.java b/android_app/app/src/test/java/dev/fanis/expensenotification/ConfigRuntimeRobolectricTest.java index da1f46e..c7ec26c 100644 --- a/android_app/app/src/test/java/dev/fanis/expensenotification/ConfigRuntimeRobolectricTest.java +++ b/android_app/app/src/test/java/dev/fanis/expensenotification/ConfigRuntimeRobolectricTest.java @@ -2,6 +2,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import android.content.Context; @@ -59,6 +60,25 @@ public void userInputOverrideIsLoadedAfterRevisionBump() throws Exception { assertEquals("Test Bank", candidate.suggestedPaymentMethod); } + @Test + public void hiddenBundledInputStopsParsingUntilRestored() { + // Deleting (hiding) a bundled config from the Config screen must actually + // stop its notifications from being captured, and restoring must bring it back. + Context context = RuntimeEnvironment.getApplication(); + String title = "SAMPLE UTILITY"; + String body = "You spent €33.95\nEUR balance: €543.21"; + + assertNotNull(ExpenseParser.parse(context, "com.revolut.revolut", "Revolut", "k", 0L, title, body)); + + ConfigHides.hide(context, "inputs", "revolut.json"); + ConfigRevision.bump(context); + assertNull(ExpenseParser.parse(context, "com.revolut.revolut", "Revolut", "k", 0L, title, body)); + + ConfigHides.restore(context, "inputs", "revolut.json"); + ConfigRevision.bump(context); + assertNotNull(ExpenseParser.parse(context, "com.revolut.revolut", "Revolut", "k", 0L, title, body)); + } + @Test public void confirmedSmsAppCanMatchSenderBasedBankConfig() { Context context = RuntimeEnvironment.getApplication(); diff --git a/android_app/app/src/test/java/dev/fanis/expensenotification/DedupeKeyTest.java b/android_app/app/src/test/java/dev/fanis/expensenotification/DedupeKeyTest.java new file mode 100644 index 0000000..a6a96e2 --- /dev/null +++ b/android_app/app/src/test/java/dev/fanis/expensenotification/DedupeKeyTest.java @@ -0,0 +1,33 @@ +package dev.fanis.expensenotification; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +import org.junit.Test; + +public class DedupeKeyTest { + + @Test + public void sameNotificationKeyAndBodyDedupeToSameKey() { + assertEquals( + ExpenseNotificationListener.dedupeKey("0|com.textra|1|null|10000", "You spent €5.00"), + ExpenseNotificationListener.dedupeKey("0|com.textra|1|null|10000", "You spent €5.00")); + } + + @Test + public void differentBodiesUnderSameNotificationKeyStayDistinct() { + // Messaging apps reuse one conversation notification (same sbn key) for every + // SMS from a sender; each distinct body must produce a distinct candidate key. + String key = "0|com.textra|1|null|10000"; + assertNotEquals( + ExpenseNotificationListener.dedupeKey(key, "You spent €5.00"), + ExpenseNotificationListener.dedupeKey(key, "You spent €6.00")); + } + + @Test + public void nullBodyIsAccepted() { + assertEquals( + ExpenseNotificationListener.dedupeKey("k", null), + ExpenseNotificationListener.dedupeKey("k", "")); + } +} diff --git a/android_app/app/src/test/java/dev/fanis/expensenotification/ExpenseParserTest.java b/android_app/app/src/test/java/dev/fanis/expensenotification/ExpenseParserTest.java index bb96811..1cf98eb 100644 --- a/android_app/app/src/test/java/dev/fanis/expensenotification/ExpenseParserTest.java +++ b/android_app/app/src/test/java/dev/fanis/expensenotification/ExpenseParserTest.java @@ -10,11 +10,16 @@ import org.json.JSONObject; import org.junit.Test; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Locale; public class ExpenseParserTest { @@ -56,8 +61,24 @@ private static String toLatinLookalike(String s) { return b.toString(); } + // The bundled asset configs are the single source of truth; the JVM tests run + // the exact JSON files that ship in the APK. + private static Candidate parseAssets(String packageName, String appName, long postedAt, String title, String body) { + try { + return ExpenseParser.parseConfigured( + readAsset("global.json"), readAllAssetInputs(), + packageName, appName, "key", postedAt, title, body); + } catch (Exception e) { + throw new AssertionError(e); + } + } + private static Candidate parseSms(String sender, String body) { - return ExpenseParser.parse(SMS_PACKAGE, SMS_APP, "key", 0L, sender, body); + return parseAssets(SMS_PACKAGE, SMS_APP, 0L, sender, body); + } + + private static Candidate parseRevolut(String title, String body) { + return parseAssets("com.revolut.business", "Business", 0L, title, body); } private static String formatDay(long postedAt) { @@ -65,9 +86,7 @@ private static String formatDay(long postedAt) { } @Test - public void bundledLocalBankDefaultsAreRegexConfigsNotLegacyHandlers() throws Exception { - assertRegexOnlyBankConfig(new JSONObject(ExpenseParser.defaultInputJson("bank-of-cyprus.json"))); - assertRegexOnlyBankConfig(new JSONObject(ExpenseParser.defaultInputJson("eurobank.json"))); + public void bundledLocalBankConfigsAreRegexConfigsNotLegacyHandlers() throws Exception { assertRegexOnlyBankConfig(new JSONObject(readAssetInput("bank-of-cyprus.json"))); assertRegexOnlyBankConfig(new JSONObject(readAssetInput("eurobank.json"))); } @@ -105,12 +124,33 @@ private static void assertRegexOnlyBankConfig(JSONObject config) throws Exceptio assertTrue("Expected at least one regex rule in " + config.optString("id"), sawRegex); } - private static String readAssetInput(String name) throws Exception { - Path path = Path.of("src", "main", "assets", "inputs", name); + private static Path assetsDir() { + Path path = Path.of("src", "main", "assets"); if (!Files.exists(path)) { - path = Path.of("app", "src", "main", "assets", "inputs", name); + path = Path.of("app", "src", "main", "assets"); + } + return path; + } + + private static String readAsset(String name) throws IOException { + return new String(Files.readAllBytes(assetsDir().resolve(name)), StandardCharsets.UTF_8); + } + + private static String readAssetInput(String name) throws IOException { + return readAsset("inputs/" + name); + } + + private static LinkedHashMap readAllAssetInputs() throws IOException { + LinkedHashMap inputs = new LinkedHashMap<>(); + List files = new ArrayList<>(); + try (java.util.stream.Stream stream = Files.list(assetsDir().resolve("inputs"))) { + stream.filter(p -> p.getFileName().toString().endsWith(".json")).forEach(files::add); + } + Collections.sort(files); + for (Path file : files) { + inputs.put(file.getFileName().toString(), new String(Files.readAllBytes(file), StandardCharsets.UTF_8)); } - return new String(Files.readAllBytes(path), StandardCharsets.UTF_8); + return inputs; } @Test @@ -118,8 +158,8 @@ public void googleWalletBillBecomesMerchantAndCreditCard() { // Real Google Wallet notification: title is the merchant, body names the // underlying card. The raw "Revolut Visa ..NNNN" descriptor is not a kept // payment method, so it must fall back to the default card. - Candidate c = ExpenseParser.parse( - "com.google.android.apps.walletnfcrel", "Google Wallet", "k", 0L, + Candidate c = parseAssets( + "com.google.android.apps.walletnfcrel", "Google Wallet", 0L, "SAMPLE BILLER", "€44.09 with Revolut Visa ••0000\nView your purchase\n100000000"); assertNotNull(c); @@ -132,8 +172,8 @@ public void googleWalletBillBecomesMerchantAndCreditCard() { @Test public void revolutSpendBecomesMerchantAndCreditCard() { // Real Revolut notification: title is the merchant, body is "You spent ...". - Candidate c = ExpenseParser.parse( - "com.revolut.revolut", "Revolut", "k", 0L, + Candidate c = parseAssets( + "com.revolut.revolut", "Revolut", 0L, "SAMPLE UTILITY", "You spent €33.95\nEUR balance: €543.21"); assertNotNull(c); @@ -288,7 +328,7 @@ public void datesExpenseToTransactionDateInSmsBody() { // The SMS arrived (notification posted) far later than the transaction; the // expense must be dated to the date written in the SMS, not the arrival time. long arrivedNov2023 = 1_700_000_000_000L; - Candidate c = ExpenseParser.parse(SMS_PACKAGE, SMS_APP, "key", arrivedNov2023, "BOC Message", + Candidate c = parseAssets(SMS_PACKAGE, SMS_APP, arrivedNov2023, "BOC Message", "Η ΚΑΡΤΑ ΣΑΣ VISA*1234 ΕΧΕΙ ΧΡΗΣΙΜΟΠΟΙΗΘΕΙ ΣΤΟ SAMPLE STORE " + "ΣΤΙΣ 26/05/2026, 20:17 ΓΙΑ ΤΟ ΕΝΔΕΙΚΤΙΚΟ ΠΟΣΟ €15,30."); assertNotNull(c); @@ -298,7 +338,7 @@ public void datesExpenseToTransactionDateInSmsBody() { @Test public void datesIncomingCreditToTwoDigitYearDate() { long arrived = 1_700_000_000_000L; - Candidate c = ExpenseParser.parse(SMS_PACKAGE, SMS_APP, "key", arrived, "BOC Message", + Candidate c = parseAssets(SMS_PACKAGE, SMS_APP, arrived, "BOC Message", "The a/c XXXX000000 (CURRENT) was credited with the amount of " + "EUR 300,00 on 23/05/26 12:55 From: SAMPLE PAYER Details: SAMPLE NOTE"); assertNotNull(c); @@ -310,7 +350,7 @@ public void keepsNotificationPostTimeWhenSmsHasNoDate() { // Revolut spend notifications carry no transaction date, so the expense keeps // the notification's post time. long arrived = 1_700_000_000_000L; - Candidate c = ExpenseParser.parse("com.revolut.revolut", "Revolut", "k", arrived, + Candidate c = parseAssets("com.revolut.revolut", "Revolut", arrived, "SAMPLE UTILITY", "You spent €33.95\nEUR balance: €543.21"); assertNotNull(c); assertEquals(arrived, c.postedAt); @@ -392,10 +432,6 @@ public void parsesEurobankCyIncomingCreditWithThousandsSeparator() { } } - private static Candidate parseRevolut(String title, String body) { - return ExpenseParser.parse("com.revolut.business", "Business", "k", 0L, title, body); - } - @Test public void rejectsPendingApprovalPrompt() { // The 3DS approval request that precedes the real "successful" notification. @@ -424,6 +460,17 @@ public void rejectsZeroAmount() { assertNull(c); } + @Test + public void dropZeroAmountFalseKeepsZeroCharges() throws Exception { + // The global dropZeroAmount switch must actually control the zero filter. + Candidate c = ExpenseParser.parseConfigured( + "{\"dropZeroAmount\":false}", readAllAssetInputs(), + "com.revolut.revolut", "Revolut", "k", 0L, + "Some Shop", "You spent €0\nEUR balance: €10.00"); + assertNotNull(c); + assertEquals("0", c.amount); + } + @Test public void keepsCompletedRevolutPayment() { // The genuine completion must still pass after the new filters. @@ -434,4 +481,32 @@ public void keepsCompletedRevolutPayment() { assertEquals("112.14", c.amount); assertEquals("SAMPLE UTILITY", c.merchant); } + + // ---- Amount normalization: European/English grouping and decimals. ---- + + @Test + public void normalizesGroupedThousandsWithoutDecimals() { + assertEquals("1234", ExpenseParser.normalizeAmount("1.234")); + assertEquals("1234", ExpenseParser.normalizeAmount("1,234")); + assertEquals("1234567", ExpenseParser.normalizeAmount("1.234.567")); + } + + @Test + public void normalizesDecimalsAndMixedSeparators() { + assertEquals("1234.56", ExpenseParser.normalizeAmount("1.234,56")); + assertEquals("1234.56", ExpenseParser.normalizeAmount("1,234.56")); + assertEquals("300.00", ExpenseParser.normalizeAmount("300,00")); + assertEquals("12.34", ExpenseParser.normalizeAmount("12.34")); + assertEquals("3.5", ExpenseParser.normalizeAmount("3,5")); + } + + @Test + public void groupedThousandsWithoutDecimalsParseAsWholeAmount() { + // "€1.234" is one thousand two hundred thirty four euros, not 1.23. + Candidate c = parseAssets("com.revolut.revolut", "Revolut", 0L, + "SAMPLE STORE", "You spent €1.234\nEUR balance: €5.678,90"); + assertNotNull(c); + assertEquals("1234", c.amount); + assertEquals("EUR", c.currency); + } } diff --git a/docs/config-schema.md b/docs/config-schema.md index 414253d..30f9d78 100644 --- a/docs/config-schema.md +++ b/docs/config-schema.md @@ -1,6 +1,14 @@ # Parser and Output Config Schema -Configs are JSON files bundled in `android_app/app/src/main/assets` and user-editable on-device from Settings > Parser and output configs. User files with the same name override bundled files. +Configs are JSON files bundled in `android_app/app/src/main/assets` and user-editable on-device from Settings > Parser and output configs. User files with the same name override bundled files. Deleting a bundled config from the UI hides it: the parser stops using it entirely until it is restored. + +## Global Config + +`global.json` holds parser-wide settings: + +- `smsPackages`: messaging app packages whose notifications are checked against SMS `senders`. Extended on-device from Settings > SMS apps. +- `rejectPhrases`: lowercase phrases that mark a notification as not-a-charge (3DS prompts, declines, card verification); matching notifications are never queued. +- `dropZeroAmount`: drop zero-amount captures (e.g. a €0 card registration). Defaults to `true`. ## Input Parser