Allowing user to add a note after creating a highlight.

README.MD

@@ -29,14 +29,15 @@ needs to be completed manually.
 
 - Create a ticket on GitHub for bug reports and feature requests.
 - Follow the [git-flow branching scheme][git-flow-instructions] (use the [git-flow CLI tool if you wish][git-flow-cli]).
-- Bump the version using `bumpversion <major|minor|patch>`.
+- Bump the version using `bumpversion <major|minor|patch>`. Versioning follows the [Semantic Versioning][semver] scheme.
 - Create a PR into `develop` and I'll merge your work in.
 
 [readwise]: https://readwise.io
 [openai]: https://openai.com
 [readwise_api_key]: https://readwise.io/access_token
 [openai_api_key]: https://beta.openai.com/account/api-keys
 [anki_addons_folder]: https://addon-docs.ankiweb.net/addon-folders.html
+[semver]: https://semver.org/
 
 [git-flow-instructions]: https://nvie.com/posts/a-successful-git-branching-model/
-[git-flow-tool]: https://github.com/nvie/gitflow/tree/master
+[git-flow-tool]: https://github.com/nvie/gitflow/tree/master

__init__.py

@@ -1,10 +1,17 @@
+import datetime
+import json
+import concurrent
+import concurrent.futures
 import anki
+from anki.collection import Collection
 
 # import the main window object (mw) from aqt
 from aqt import mw, gui_hooks
+
 # import the "show info" tool from utils.py
 from aqt.utils import showInfo, qconnect
-from aqt.operations import QueryOp
+from aqt.operations import CollectionOp
+
 # import all of the Qt GUI library
 from aqt.qt import *
 
@@ -22,113 +29,138 @@
 import openai  # noqa: E402
 
 from .readwise import ReadwiseClient
-from .logging_utils import make_logger
+from .logging_utils import make_logger, log_exceptions
+from .notetype import SmoothBrainNotetype
+from .config import Config
 
+# TODO: Let users define the log level in the config
 LOG_FILE = os.path.join(ADDON_ROOT_DIR, f"{__name__}.log")
 logger = make_logger(__name__, filepath=LOG_FILE)
 
-config = mw.addonManager.getConfig(__name__)
-OPENAI_API_KEY = config["openai_api_key"]
-READWISE_API_KEY = config["readwise_api_key"]
-DECK_NAME = config["deck_name"]
-
-OPENAI_DEFAULT_MODEL = "text-davinci-003"
-OPENAI_MAX_TOKENS = 4096
-OPENAI_MAX_OUTPUT_TOKENS = 256
-openai.api_key = OPENAI_API_KEY
-openai.api_base = config.get("openai_base_url", "https://oai.hconeai.com/v1")  # Helicone for stats
-
-
-# We're going to add a menu item below. First we want to create a function to
-# be called when the menu item is activated.
-def get_ai_flashcards_for_doc(doc):
-    # TODO: give pos/neg examples of what it gives me but what I actually want
-    # TODO: Try using Curie / Davinci with fine-tuning
-    # TODO: Handle list/composite highlights
-    # TODO: Add retry logic, only surface error after a few tries with backoff
-    # TODO: Let them be bad but let user re-gen it with a prompt. Save prompt
-    prompt_template = f"""
-    Make a succinct flash card for the following:
-
-    {{}}
-
-    Remember to:
-    1. Be straight to the point.
-    2. Only test ONE fact.
-    3. Prefer Q&A format.
-    """
-    responses = [complete(prompt_template.format(h.text)) for h in doc.highlights]
-    return responses
+config = Config(mw.addonManager)
 
-def query_for_ai_flashcards(doc):
-    return MyQueryOp(
-        parent=mw,
-        op=lambda col: (doc, get_ai_flashcards_for_doc(doc)),
-    )
+OPENAI_DEFAULT_MODEL = "gpt-4"
 
-def identity_function(*args):
-    return args
 
-class MyQueryOp:
-    def __init__(self, parent, op):
-        self._parent = parent
-        self._op = op
-        self._success = identity_function
+# TODO: Make this a decorator so it can reset the value after the function completes (or if it errors)
+def set_openai_api_parameters(config):
+    openai.api_key = config["openai_api_key"]
+    openai.api_base = config.get("openai_api_base", openai.api_base)
 
-    def op(self):
-        return QueryOp(parent=self._parent, op=self._op, success=self._success)
-
-    def success(self, success):
-        self._success = success
-        return self
-
-    def run_in_background(self):
-        self.op().run_in_background()
 
-def sync_readwise() -> None:
-    return MyQueryOp(
-        parent=mw,
-        op=lambda col: get_filtered_readwise_highlights(),
-    )
+def do_sync():
+    @log_exceptions(logger)
+    def op(col: Collection):
+        # TODO: Get latest fetch time from deck instead of config (what if we delete the deck?)
+        # TODO: Remove duplicate or VERY similar cards even if from different highlights
+        notetype = SmoothBrainNotetype(col)
+        want_cancel = False
 
-def make_flashcard(doc, highlight, openai_response):
-    pass
+        def update_progress(label, value=None, max=None):
+            def cb():
+                mw.progress.update(label=label, value=value, max=max)
+                nonlocal want_cancel
+                want_cancel = mw.progress.want_cancel()
+
+            mw.taskman.run_on_main(cb)
+
+        undo_entry = col.add_custom_undo_entry("Sync Readwise")
+        docs = get_filtered_readwise_highlights()
+        docs = (
+            docs[: config["max_num_docs_to_fetch"]]
+            if "max_num_docs_to_fetch" in config
+            else docs
+        )
+
+        # TODO: Wait until flashcards are generated before adding them to the deck.
+        # Should probably use an SQLite database to store partial results so they don't
+        # pollute the Anki database.
+        deck_id = col.decks.add_normal_deck_with_name(config["deck_name"]).id
+        notes = []
+        executor = concurrent.futures.ThreadPoolExecutor(max_workers=4)
+        future_to_note = {}
+        try:
+            for i, doc in enumerate(docs, start=1):
+                if want_cancel:
+                    break
+                update_progress(
+                    f"Fetching Readwise document {i} of {len(docs)}...",
+                    value=i - 1,
+                    max=len(docs),
+                )
+                for hl in doc.highlights:
+                    note, added = notetype.get_or_create(doc, hl)
+                    notes.append(note)
+                    if added:
+                        future = executor.submit(
+                            lambda h: complete(
+                                h.text, config.get("openai_model", OPENAI_DEFAULT_MODEL)
+                            )
+                            .choices[0]["message"]["content"]
+                            .strip(),
+                            hl,
+                        )
+                        future_to_note[future] = note
+                        col.add_note(note=note, deck_id=deck_id)
+                    # Merge to our custom undo entry before the undo queue fills up and Anki discards our entry
+                    if (col.undo_status().last_step - undo_entry) % 29 == 0:
+                        col.merge_undo_entries(undo_entry)
+
+            for i, future in enumerate(
+                concurrent.futures.as_completed(future_to_note), start=1
+            ):
+                if want_cancel:
+                    break
+                update_progress(
+                    f"Generating questions for highlight {i} of {len(future_to_note.keys())}...",
+                    value=i - 1,
+                    max=len(future_to_note.keys()),
+                )
+                completion = future.result()
+                try:
+                    note = future_to_note[future]
+                    result = json.loads(completion)
+                    if not result:
+                        col.sched.suspend_cards([n.id for n in note.cards()])
+                        continue
+                    # TODO: Use all of the responses, and don't add a note if it doesn't have a flashcard.
+                    note["question"] = result[0]["question"]
+                    note["answer"] = result[0]["answer"]
+                except json.decoder.JSONDecodeError as e:
+                    logger.error(
+                        f"Failed to parse completion as JSON. Completion: {completion}"
+                    )
+                    raise e
+                except ValueError as e:
+                    logger.error(
+                        f"Failed to split completion into question and answer. Result: {result}"
+                    )
+                    raise e
+        finally:
+            col.update_notes(notes)
+        return col.merge_undo_entries(undo_entry)
+
+    CollectionOp(parent=mw, op=op).run_in_background()
 
-def do_sync():
-    # TODO: Use promises instead of callbacks
-    def make_deck(docs):
-        from aqt.operations.deck import add_deck
-        # TODO: Only add a deck if the cards don't already exist
-        def generate_flashcards(deck_id):
-            def update_card(result):
-                from aqt.operations.note import add_note
-                # TODO: Create a function that accepts a deck_id, looks for card ids, etc...
-                # TODO: Search how to create a note
-                # docs: list[list[openai_response]] (one for each highlight)
-                # Add a note with docs[0][0].choices[0].text
-                #note = None
-                #add_note(parent=mw, note=note, target_deck_id=deck_id)
-                doc, completions = result
-                completions = [c.choices[0].text.strip() for c in completions]
-                for hl, completion in zip(doc.highlights, completions):
-                    question, answer = completion.split("A:")
-                    question = question[len("Q: "):]
-                    model = mw.col.models.by_name("Basic")
-                    note = mw.col.new_note(model)
-                    note["Front"] = question
-                    note["Back"] = answer
-                    # TODO: Use a single CollectionOp to create notes instead of multiple
-                    add_note(parent=mw, note=note, target_deck_id=deck_id.id).run_in_background()
-            for doc in docs[:1]:
-                query_for_ai_flashcards(doc).success(update_card).run_in_background()
-        # TODO: Make the deck have a certain template
-        add_deck(parent=mw, name=DECK_NAME).success(generate_flashcards).run_in_background()
-    sync_readwise().success(make_deck).run_in_background()
 
+@log_exceptions(logger)
 def get_filtered_readwise_highlights():
-    readwise_client = ReadwiseClient(api_key=READWISE_API_KEY).set_parent_logger(logger)
-    docs = readwise_client.export()
+    latest_fetch_time = (
+        datetime.datetime.fromisoformat(config["latest_fetch_time"])
+        if config["latest_fetch_time"]
+        else None
+    )
+    readwise_client = (
+        ReadwiseClient(api_key=config["readwise_api_key"])
+        .set_parent_logger(logger)
+        .set_latest_fetch_time(latest_fetch_time)
+    )
+    docs = readwise_client.updates()
+    config["latest_fetch_time"] = datetime.datetime.isoformat(
+        readwise_client.latest_fetch_time
+    )
     sources_to_ignore = {
+        # TODO: Allow these to be configured
         # Things that we didn't highlight. Readwise adds
         # supplemental popular highlights from things we've read,
         # which is nice, but I think people should be intentional
@@ -149,26 +181,66 @@ def get_filtered_readwise_highlights():
         "twitter",
     }
     filtered_highlights = [
-        d for d in docs
+        d
+        for d in docs
         if d.source not in sources_to_ignore
         # Only fetch highlights
-        # TODO: Add support for x["document_note"]
         if d.highlights
+        # TODO: Filter tags
     ]
     return filtered_highlights
 
 
-# TODO: Use backoff and/or rate-limit
-# TODO: Allow these parameters to be customized in advanced menu
-def complete(prompt):
-    return openai.Completion.create(engine=OPENAI_DEFAULT_MODEL,
-                                    prompt=prompt,
-                                    max_tokens=OPENAI_MAX_OUTPUT_TOKENS,
-                                    temperature=0.5,
-                                    top_p=1,
-                                    frequency_penalty=0,
-                                    presence_penalty=0)
+# TODO: Use backoff and/or rate-limit.
+# TODO: Ask GPT to add a reason if there are no questions, for debugging.
+# TODO: Generate questions for entire doc, and fetch the entire doc if possible to improve the context
+@log_exceptions(logger)
+def complete(prompt, model):
+    set_openai_api_parameters(config)
+    # https://platform.openai.com/playground?mode=chat&model=gpt-4 is a great way to test this.
+
+    system = """You are an advanced AI programmed to generate educational flashcards for active recall learning. Your role is to extract and enhance key facts from provided text and present them in a succinct, educational format. Here’s your task breakdown:
+
+1. **Summarization & Enrichment**: Condense and enrich long texts into compact paragraphs, ensuring the essence and educational value of the content are preserved.
+2. **Cloze Deletion Strategy**: When crafting cloze deletions, focus on both the key concept and its definition. Ensure that:
+    - The key concept is one cloze deletion.
+    - The definition or description of the concept is broken down into one or more additional cloze deletions, that can have auxiliary words in the middle, for example the same cloze deletion c2 is splited: [ {"question": "In statistics, the {{c1::range}} of a sample is calculated as the {{c2::maximum}} minus the {{c2::minimum}}, which is denoted as {{c3::x(n) – x(1)}}.", "answer": "In statistics, the range of a sample is calculated as the maximum minus the minimum, which is denoted as x(n) – x(1)."} ]
+3. **Flashcard Structuring**: Compose each fact as a JSON object with "question" and "answer" fields. The "question" should contain the text with cloze deletions, while the "answer" presents the complete fact.
+4. **Contextual Relevance & Brevity**: Provide sufficient context to make the flashcard meaningful. Keep the flashcards concise, focusing on the core information.
+5. **Cloze Deletion Optimization**: Aim for at least two and maximum three cloze deletions per flashcard, ensuring that both the concept and its definition are adequately cloze deleted for effective reciprocal recall.
+6. **Factual Accuracy & Clarity**: Ensure the flashcards are factually accurate and clearly presented.
+
+Example:
+
+Good example: Input: "A rate ratio is the relative increase in the expected number of events in a fixed period of time associated with an exposure." Output: [ {"question": "A {{c1::rate ratio}} is the {{c2::relative increase in the expected number of events}} in a {{c2::fixed period of time}} associated with {{c3::an exposure}}.", "answer": "A rate ratio is the relative increase in the expected number of events in a fixed period of time associated with an exposure."} ]
+
+Key Guidelines:
 
+- Ensure the context in the question sufficiently supports the answer.
+- Facts used should be verifiable and accurate.
+- Cloze delete both the concept and its definition for reciprocal learning.
+- Expand on the information if necessary, but maintain conciseness in flashcards.
+
+    """
+
+
+    completion = openai.ChatCompletion.create(
+        model=model,
+        messages=[
+            {"role": "system", "content": system},
+            {"role": "user", "content": prompt},
+        ],
+        # TODO: Allow these parameters to be customized in config
+        temperature=0,
+        n=1,
+        headers={
+            "Helicone-Cache-Enabled": "true",
+        },
+    )
+    return completion
+
+
+@log_exceptions(logger)
 def setup_menu():
     # TODO: Pass in top level menu and derive window from it
     # Create the menu button
@@ -178,24 +250,12 @@ def setup_menu():
     action.setShortcut(QKeySequence("Ctrl+R"))
     mw.form.menuTools.addAction(action)
 
-def setup_hooks():
-    gui_hooks.sync_did_finish.append(do_sync)
 
-#setup_hooks()
+# TODO: Consider automating syncing
+# @log_exceptions(logger)
+# def setup_hooks():
+#    gui_hooks.sync_did_finish.append(do_sync)
+# setup_hooks()
 
-if (QAction != None and mw != None):
+if QAction != None and mw != None:
     setup_menu()
-    #mw.form.menuTool
-    #setup_menu(
-
-"""
-TODO:
-- Create flashcards in deck
--- Custom card type? Just do Q&A at first, then add fields.
-- Cache Readwise results
-- Store last-fetch date (to reduce query to readwise/service)
-- Make Flask backend in Replit in order to support fine-tuning/subscription
-- Config screen
-- Refactor
-"""
-

config.json

@@ -2,5 +2,8 @@
   "deck_name": "Readwise Highlights",
   "readwise_api_key": "",
   "openai_api_key": "",
-  "openai_base_url": "https://oai.hconeai.com/v1"
+  "openai_api_base": "https://api.openai.com/v1",
+  "openai_model": "gpt-3.5-turbo",
+  "latest_fetch_time": null,
+  "max_num_docs_to_fetch": null
 }