diff --git a/README.MD b/README.MD index 841be1b..1d6a89d 100644 --- a/README.MD +++ b/README.MD @@ -29,7 +29,7 @@ needs to be completed manually. - Create a ticket on GitHub for bug reports and feature requests. - Follow the [git-flow branching scheme][git-flow-instructions] (use the [git-flow CLI tool if you wish][git-flow-cli]). -- Bump the version using `bumpversion `. +- Bump the version using `bumpversion `. Versioning follows the [Semantic Versioning][semver] scheme. - Create a PR into `develop` and I'll merge your work in. [readwise]: https://readwise.io @@ -37,6 +37,7 @@ needs to be completed manually. [readwise_api_key]: https://readwise.io/access_token [openai_api_key]: https://beta.openai.com/account/api-keys [anki_addons_folder]: https://addon-docs.ankiweb.net/addon-folders.html +[semver]: https://semver.org/ [git-flow-instructions]: https://nvie.com/posts/a-successful-git-branching-model/ -[git-flow-tool]: https://github.com/nvie/gitflow/tree/master \ No newline at end of file +[git-flow-tool]: https://github.com/nvie/gitflow/tree/master diff --git a/__init__.py b/__init__.py index ed2e821..acd8fa4 100644 --- a/__init__.py +++ b/__init__.py @@ -1,10 +1,17 @@ +import datetime +import json +import concurrent +import concurrent.futures import anki +from anki.collection import Collection # import the main window object (mw) from aqt from aqt import mw, gui_hooks + # import the "show info" tool from utils.py from aqt.utils import showInfo, qconnect -from aqt.operations import QueryOp +from aqt.operations import CollectionOp + # import all of the Qt GUI library from aqt.qt import * @@ -22,113 +29,138 @@ import openai # noqa: E402 from .readwise import ReadwiseClient -from .logging_utils import make_logger +from .logging_utils import make_logger, log_exceptions +from .notetype import SmoothBrainNotetype +from .config import Config +# TODO: Let users define the log level in the config LOG_FILE = os.path.join(ADDON_ROOT_DIR, f"{__name__}.log") logger = make_logger(__name__, filepath=LOG_FILE) -config = mw.addonManager.getConfig(__name__) -OPENAI_API_KEY = config["openai_api_key"] -READWISE_API_KEY = config["readwise_api_key"] -DECK_NAME = config["deck_name"] - -OPENAI_DEFAULT_MODEL = "text-davinci-003" -OPENAI_MAX_TOKENS = 4096 -OPENAI_MAX_OUTPUT_TOKENS = 256 -openai.api_key = OPENAI_API_KEY -openai.api_base = config.get("openai_base_url", "https://oai.hconeai.com/v1") # Helicone for stats - - -# We're going to add a menu item below. First we want to create a function to -# be called when the menu item is activated. -def get_ai_flashcards_for_doc(doc): - # TODO: give pos/neg examples of what it gives me but what I actually want - # TODO: Try using Curie / Davinci with fine-tuning - # TODO: Handle list/composite highlights - # TODO: Add retry logic, only surface error after a few tries with backoff - # TODO: Let them be bad but let user re-gen it with a prompt. Save prompt - prompt_template = f""" - Make a succinct flash card for the following: - - {{}} - - Remember to: - 1. Be straight to the point. - 2. Only test ONE fact. - 3. Prefer Q&A format. - """ - responses = [complete(prompt_template.format(h.text)) for h in doc.highlights] - return responses +config = Config(mw.addonManager) -def query_for_ai_flashcards(doc): - return MyQueryOp( - parent=mw, - op=lambda col: (doc, get_ai_flashcards_for_doc(doc)), - ) +OPENAI_DEFAULT_MODEL = "gpt-4" -def identity_function(*args): - return args -class MyQueryOp: - def __init__(self, parent, op): - self._parent = parent - self._op = op - self._success = identity_function +# TODO: Make this a decorator so it can reset the value after the function completes (or if it errors) +def set_openai_api_parameters(config): + openai.api_key = config["openai_api_key"] + openai.api_base = config.get("openai_api_base", openai.api_base) - def op(self): - return QueryOp(parent=self._parent, op=self._op, success=self._success) - - def success(self, success): - self._success = success - return self - - def run_in_background(self): - self.op().run_in_background() -def sync_readwise() -> None: - return MyQueryOp( - parent=mw, - op=lambda col: get_filtered_readwise_highlights(), - ) +def do_sync(): + @log_exceptions(logger) + def op(col: Collection): + # TODO: Get latest fetch time from deck instead of config (what if we delete the deck?) + # TODO: Remove duplicate or VERY similar cards even if from different highlights + notetype = SmoothBrainNotetype(col) + want_cancel = False -def make_flashcard(doc, highlight, openai_response): - pass + def update_progress(label, value=None, max=None): + def cb(): + mw.progress.update(label=label, value=value, max=max) + nonlocal want_cancel + want_cancel = mw.progress.want_cancel() + + mw.taskman.run_on_main(cb) + + undo_entry = col.add_custom_undo_entry("Sync Readwise") + docs = get_filtered_readwise_highlights() + docs = ( + docs[: config["max_num_docs_to_fetch"]] + if "max_num_docs_to_fetch" in config + else docs + ) + + # TODO: Wait until flashcards are generated before adding them to the deck. + # Should probably use an SQLite database to store partial results so they don't + # pollute the Anki database. + deck_id = col.decks.add_normal_deck_with_name(config["deck_name"]).id + notes = [] + executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) + future_to_note = {} + try: + for i, doc in enumerate(docs, start=1): + if want_cancel: + break + update_progress( + f"Fetching Readwise document {i} of {len(docs)}...", + value=i - 1, + max=len(docs), + ) + for hl in doc.highlights: + note, added = notetype.get_or_create(doc, hl) + notes.append(note) + if added: + future = executor.submit( + lambda h: complete( + h.text, config.get("openai_model", OPENAI_DEFAULT_MODEL) + ) + .choices[0]["message"]["content"] + .strip(), + hl, + ) + future_to_note[future] = note + col.add_note(note=note, deck_id=deck_id) + # Merge to our custom undo entry before the undo queue fills up and Anki discards our entry + if (col.undo_status().last_step - undo_entry) % 29 == 0: + col.merge_undo_entries(undo_entry) + + for i, future in enumerate( + concurrent.futures.as_completed(future_to_note), start=1 + ): + if want_cancel: + break + update_progress( + f"Generating questions for highlight {i} of {len(future_to_note.keys())}...", + value=i - 1, + max=len(future_to_note.keys()), + ) + completion = future.result() + try: + note = future_to_note[future] + result = json.loads(completion) + if not result: + col.sched.suspend_cards([n.id for n in note.cards()]) + continue + # TODO: Use all of the responses, and don't add a note if it doesn't have a flashcard. + note["question"] = result[0]["question"] + note["answer"] = result[0]["answer"] + except json.decoder.JSONDecodeError as e: + logger.error( + f"Failed to parse completion as JSON. Completion: {completion}" + ) + raise e + except ValueError as e: + logger.error( + f"Failed to split completion into question and answer. Result: {result}" + ) + raise e + finally: + col.update_notes(notes) + return col.merge_undo_entries(undo_entry) + + CollectionOp(parent=mw, op=op).run_in_background() -def do_sync(): - # TODO: Use promises instead of callbacks - def make_deck(docs): - from aqt.operations.deck import add_deck - # TODO: Only add a deck if the cards don't already exist - def generate_flashcards(deck_id): - def update_card(result): - from aqt.operations.note import add_note - # TODO: Create a function that accepts a deck_id, looks for card ids, etc... - # TODO: Search how to create a note - # docs: list[list[openai_response]] (one for each highlight) - # Add a note with docs[0][0].choices[0].text - #note = None - #add_note(parent=mw, note=note, target_deck_id=deck_id) - doc, completions = result - completions = [c.choices[0].text.strip() for c in completions] - for hl, completion in zip(doc.highlights, completions): - question, answer = completion.split("A:") - question = question[len("Q: "):] - model = mw.col.models.by_name("Basic") - note = mw.col.new_note(model) - note["Front"] = question - note["Back"] = answer - # TODO: Use a single CollectionOp to create notes instead of multiple - add_note(parent=mw, note=note, target_deck_id=deck_id.id).run_in_background() - for doc in docs[:1]: - query_for_ai_flashcards(doc).success(update_card).run_in_background() - # TODO: Make the deck have a certain template - add_deck(parent=mw, name=DECK_NAME).success(generate_flashcards).run_in_background() - sync_readwise().success(make_deck).run_in_background() +@log_exceptions(logger) def get_filtered_readwise_highlights(): - readwise_client = ReadwiseClient(api_key=READWISE_API_KEY).set_parent_logger(logger) - docs = readwise_client.export() + latest_fetch_time = ( + datetime.datetime.fromisoformat(config["latest_fetch_time"]) + if config["latest_fetch_time"] + else None + ) + readwise_client = ( + ReadwiseClient(api_key=config["readwise_api_key"]) + .set_parent_logger(logger) + .set_latest_fetch_time(latest_fetch_time) + ) + docs = readwise_client.updates() + config["latest_fetch_time"] = datetime.datetime.isoformat( + readwise_client.latest_fetch_time + ) sources_to_ignore = { + # TODO: Allow these to be configured # Things that we didn't highlight. Readwise adds # supplemental popular highlights from things we've read, # which is nice, but I think people should be intentional @@ -149,26 +181,66 @@ def get_filtered_readwise_highlights(): "twitter", } filtered_highlights = [ - d for d in docs + d + for d in docs if d.source not in sources_to_ignore # Only fetch highlights - # TODO: Add support for x["document_note"] if d.highlights + # TODO: Filter tags ] return filtered_highlights -# TODO: Use backoff and/or rate-limit -# TODO: Allow these parameters to be customized in advanced menu -def complete(prompt): - return openai.Completion.create(engine=OPENAI_DEFAULT_MODEL, - prompt=prompt, - max_tokens=OPENAI_MAX_OUTPUT_TOKENS, - temperature=0.5, - top_p=1, - frequency_penalty=0, - presence_penalty=0) +# TODO: Use backoff and/or rate-limit. +# TODO: Ask GPT to add a reason if there are no questions, for debugging. +# TODO: Generate questions for entire doc, and fetch the entire doc if possible to improve the context +@log_exceptions(logger) +def complete(prompt, model): + set_openai_api_parameters(config) + # https://platform.openai.com/playground?mode=chat&model=gpt-4 is a great way to test this. + + system = """You are an advanced AI programmed to generate educational flashcards for active recall learning. Your role is to extract and enhance key facts from provided text and present them in a succinct, educational format. Here’s your task breakdown: + +1. **Summarization & Enrichment**: Condense and enrich long texts into compact paragraphs, ensuring the essence and educational value of the content are preserved. +2. **Cloze Deletion Strategy**: When crafting cloze deletions, focus on both the key concept and its definition. Ensure that: + - The key concept is one cloze deletion. + - The definition or description of the concept is broken down into one or more additional cloze deletions, that can have auxiliary words in the middle, for example the same cloze deletion c2 is splited: [ {"question": "In statistics, the {{c1::range}} of a sample is calculated as the {{c2::maximum}} minus the {{c2::minimum}}, which is denoted as {{c3::x(n) – x(1)}}.", "answer": "In statistics, the range of a sample is calculated as the maximum minus the minimum, which is denoted as x(n) – x(1)."} ] +3. **Flashcard Structuring**: Compose each fact as a JSON object with "question" and "answer" fields. The "question" should contain the text with cloze deletions, while the "answer" presents the complete fact. +4. **Contextual Relevance & Brevity**: Provide sufficient context to make the flashcard meaningful. Keep the flashcards concise, focusing on the core information. +5. **Cloze Deletion Optimization**: Aim for at least two and maximum three cloze deletions per flashcard, ensuring that both the concept and its definition are adequately cloze deleted for effective reciprocal recall. +6. **Factual Accuracy & Clarity**: Ensure the flashcards are factually accurate and clearly presented. + +Example: + +Good example: Input: "A rate ratio is the relative increase in the expected number of events in a fixed period of time associated with an exposure." Output: [ {"question": "A {{c1::rate ratio}} is the {{c2::relative increase in the expected number of events}} in a {{c2::fixed period of time}} associated with {{c3::an exposure}}.", "answer": "A rate ratio is the relative increase in the expected number of events in a fixed period of time associated with an exposure."} ] + +Key Guidelines: +- Ensure the context in the question sufficiently supports the answer. +- Facts used should be verifiable and accurate. +- Cloze delete both the concept and its definition for reciprocal learning. +- Expand on the information if necessary, but maintain conciseness in flashcards. + + """ + + + completion = openai.ChatCompletion.create( + model=model, + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": prompt}, + ], + # TODO: Allow these parameters to be customized in config + temperature=0, + n=1, + headers={ + "Helicone-Cache-Enabled": "true", + }, + ) + return completion + + +@log_exceptions(logger) def setup_menu(): # TODO: Pass in top level menu and derive window from it # Create the menu button @@ -178,24 +250,12 @@ def setup_menu(): action.setShortcut(QKeySequence("Ctrl+R")) mw.form.menuTools.addAction(action) -def setup_hooks(): - gui_hooks.sync_did_finish.append(do_sync) -#setup_hooks() +# TODO: Consider automating syncing +# @log_exceptions(logger) +# def setup_hooks(): +# gui_hooks.sync_did_finish.append(do_sync) +# setup_hooks() -if (QAction != None and mw != None): +if QAction != None and mw != None: setup_menu() - #mw.form.menuTool - #setup_menu( - -""" -TODO: -- Create flashcards in deck --- Custom card type? Just do Q&A at first, then add fields. -- Cache Readwise results -- Store last-fetch date (to reduce query to readwise/service) -- Make Flask backend in Replit in order to support fine-tuning/subscription -- Config screen -- Refactor -""" - diff --git a/config.json b/config.json index 990bc26..675f836 100644 --- a/config.json +++ b/config.json @@ -2,5 +2,8 @@ "deck_name": "Readwise Highlights", "readwise_api_key": "", "openai_api_key": "", - "openai_base_url": "https://oai.hconeai.com/v1" + "openai_api_base": "https://api.openai.com/v1", + "openai_model": "gpt-3.5-turbo", + "latest_fetch_time": null, + "max_num_docs_to_fetch": null } diff --git a/config.py b/config.py new file mode 100644 index 0000000..82c29f9 --- /dev/null +++ b/config.py @@ -0,0 +1,34 @@ +from aqt.addons import AddonManager + + +class Config: + """A manager for config.json loading/updating/saving""" + + def __init__(self, addon_manager: AddonManager, addon_name: str = __name__): + self._addon_manager = addon_manager + self._addon_name = addon_name + self._addon_manager.setConfigUpdatedAction( + self._addon_name, self._config_updated_action + ) + + def _config(self): + return self._addon_manager.getConfig(self._addon_name) + + def _config_updated_action(self, new_config) -> None: + self._config().update(new_config) + + def _write(self) -> None: + self._addon_manager.writeConfig(self._addon_name, self._config()) + + def __getitem__(self, key): + return self._config()[key] + + def __setitem__(self, key, value): + self._config()[key] = value + self._write() + + def __contains__(self, key): + return key in self._config() + + def get(self, key, default=None): + return self._config().get(key, default) diff --git a/logging_utils.py b/logging_utils.py index 3d94ec6..11edebb 100644 --- a/logging_utils.py +++ b/logging_utils.py @@ -1,10 +1,14 @@ -import logging import json +import logging + +import functools as ft + +DEFAULT_LOG_LEVEL = logging.DEBUG -DEFAULT_LEVEL = logging.DEBUG class JsonFormatter(logging.Formatter): """A JSON formatter can be used parse logs more easily.""" + def format(self, record): log_record = { "timestamp": record.created, @@ -23,13 +27,28 @@ def make_logger(name, filepath=None, level=None): filepath = filepath or f"{name}.log" file_handler = logging.FileHandler(filepath) - #formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s: %(message)s') + # formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s: %(message)s') formatter = JsonFormatter() file_handler.setFormatter(formatter) logger.addHandler(file_handler) - level = level or DEFAULT_LEVEL + level = level or DEFAULT_LOG_LEVEL logger.setLevel(level) logger.info("Logger ready!") - return logger \ No newline at end of file + return logger + + +def log_exceptions(logger): + def decorator(f): + @ft.wraps(f) + def wrapped_f(*args, **kwargs): + try: + return f(*args, **kwargs) + except Exception as e: + logger.exception(e, exc_info=e) + raise e + + return wrapped_f + + return decorator diff --git a/notetype.py b/notetype.py new file mode 100644 index 0000000..648c51a --- /dev/null +++ b/notetype.py @@ -0,0 +1,146 @@ + +import re +from typing import List, Tuple + +from anki.collection import Collection, SearchNode +from anki.models import NotetypeDict +from anki.notes import Note +from anki.stdmodels import get_stock_notetypes +from markdown import markdown + +from .readwise import ReadwiseDocument, ReadwiseHighlight + + +class SmoothBrainBasicTemplate: + name = "Cloze" + question = """{{cloze:Question}}""" + answer = """{{cloze:Answer}}""" + + +class SmoothBrainNotetype: + name = "SmoothBrain" + templates = [SmoothBrainBasicTemplate] + fields = [ + "id", + "question", + "answer", + # Highlight fields + "text", + "note", + "url", + "readwise_url", + "highlighted_at", + "created_at", + "updated_at", + "location", + "end_location", + "color", + "is_favorite", + "is_discard", + # Document fields + "user_book_id", + "readable_title", + "title", + "document_note", + "document_readwise_url", + "author", + "source", + "source_url", + "unique_url", + "cover_image_url", + "category", + "asin", + ] + + def __init__(self, col: Collection) -> None: + self.col = col + basic_notetype = get_stock_notetypes(col)[1][1](col) + self.css = basic_notetype["css"] + self.notetype = self._ensure_exists() + + def _ensure_exists(self) -> NotetypeDict: + notetype = self.col.models.by_name(self.name) + if not notetype: + notetype = self.col.models.new(self.name) + for readwise_template in self.templates: + template = self.col.models.new_template(readwise_template.name) + template["qfmt"] = readwise_template.question + template["afmt"] = readwise_template.answer + self.col.models.add_template(notetype, template) + for field_name in self.fields: + field = self.col.models.new_field(field_name) + self.col.models.add_field(notetype, field) + notetype["css"] = self.css + self.col.models.set_sort_index(notetype, self.fields.index("question")) + self.col.models.add_dict(notetype) + # We need to refetch the notetype after adding it + notetype = self.col.models.by_name(self.name) + return notetype + + def _format_field(self, contents) -> str: + text = "" + if contents: + text = str(contents) + return text + + def _format_url(self, contents) -> str: + url = self._format_field(contents) + if url: + url = f'{url}' + return url + + def _format_image(self, contents) -> str: + url = self._format_field(contents) + if url: + url = f'' + return url + + def get_or_create( + self, doc: ReadwiseDocument, highlight: ReadwiseHighlight + ) -> Tuple[Note, bool]: + nids = self.col.find_notes( + self.col.build_search_string( + SearchNode(note=self.name), f'"id:{highlight.id}"' + ) + ) + if nids: + note = self.col.get_note(nids[0]) + added = False + else: + note = self.col.new_note(self.notetype) + added = True + + # note["Question"] = self._format_field(f"{{c1::{highlight.text}}}") + # note["Answer"] = self._format_field(highlight.text) + note["id"] = self._format_field(highlight.id) + note["text"] = markdown(self._format_field(highlight.text)) + #note["text"] = self._format_field(f"{{c1::{highlight.text}}}") + note["note"] = self._format_field(highlight.note) + note["url"] = self._format_url(highlight.url) + note["readwise_url"] = self._format_url(highlight.readwise_url) + note["highlighted_at"] = self._format_field(highlight.highlighted_at) + note["created_at"] = self._format_field(highlight.created_at) + note["updated_at"] = self._format_field(highlight.updated_at) + note["location"] = self._format_field(highlight.location) + note["end_location"] = self._format_field(highlight.end_location) + note["color"] = self._format_field(highlight.color) + note["is_favorite"] = self._format_field(highlight.is_favorite) + note["is_discard"] = self._format_field(highlight.is_discard) + note["user_book_id"] = self._format_field(doc.user_book_id) + note["readable_title"] = self._format_field(doc.readable_title) + note["title"] = self._format_field(doc.title) + note["document_note"] = self._format_field(doc.document_note) + note["document_readwise_url"] = self._format_url(doc.readwise_url) + note["author"] = self._format_field(doc.author) + note["source"] = self._format_field(doc.source) + note["source_url"] = self._format_url(doc.source_url) + note["unique_url"] = self._format_url(doc.unique_url) + note["cover_image_url"] = self._format_image(doc.cover_image_url) + note["category"] = self._format_field(doc.category) + note["asin"] = self._format_field(doc.asin) + note.tags = [tag["name"] for tag in doc.book_tags + highlight.tags] + + return note, added + + + diff --git a/notetype_original.py b/notetype_original.py new file mode 100644 index 0000000..7080079 --- /dev/null +++ b/notetype_original.py @@ -0,0 +1,148 @@ +from typing import Tuple + +from anki.collection import Collection, SearchNode +from anki.models import NotetypeDict +from anki.notes import Note +from anki.stdmodels import get_stock_notetypes +from markdown import markdown + +from .readwise import ReadwiseDocument, ReadwiseHighlight + + +# class SmoothBrainBasicTemplate: +# name = "Card 1" +# question = """{{question}}""" +# answer = """{{FrontSide}} + +#
+ +# {{answer}}""" + +class SmoothBrainBasicTemplate: + name = "Cloze" + question = """{{cloze:Question}}""" + answer = """{{cloze:Answer}}""" + + +class SmoothBrainNotetype: + name = "SmoothBrain" + templates = [SmoothBrainBasicTemplate] + fields = [ + "id", + "question", + "answer", + # Highlight fields + "text", + "note", + "url", + "readwise_url", + "highlighted_at", + "created_at", + "updated_at", + "location", + "end_location", + "color", + "is_favorite", + "is_discard", + # Document fields + "user_book_id", + "readable_title", + "title", + "document_note", + "document_readwise_url", + "author", + "source", + "source_url", + "unique_url", + "cover_image_url", + "category", + "asin", + ] + + def __init__(self, col: Collection) -> None: + self.col = col + basic_notetype = get_stock_notetypes(col)[0][1](col) + self.css = basic_notetype["css"] + self.notetype = self._ensure_exists() + + def _ensure_exists(self) -> NotetypeDict: + notetype = self.col.models.by_name(self.name) + if not notetype: + notetype = self.col.models.new(self.name) + for readwise_template in self.templates: + template = self.col.models.new_template(readwise_template.name) + template["qfmt"] = readwise_template.question + template["afmt"] = readwise_template.answer + self.col.models.add_template(notetype, template) + for field_name in self.fields: + field = self.col.models.new_field(field_name) + self.col.models.add_field(notetype, field) + notetype["css"] = self.css + self.col.models.set_sort_index(notetype, self.fields.index("question")) + self.col.models.add_dict(notetype) + # We need to refetch the notetype after adding it + notetype = self.col.models.by_name(self.name) + return notetype + + def _format_field(self, contents) -> str: + text = "" + if contents: + text = str(contents) + return text + + def _format_url(self, contents) -> str: + url = self._format_field(contents) + if url: + url = f'{url}' + return url + + def _format_image(self, contents) -> str: + url = self._format_field(contents) + if url: + url = f'' + return url + + def get_or_create( + self, doc: ReadwiseDocument, highlight: ReadwiseHighlight + ) -> Tuple[Note, bool]: + nids = self.col.find_notes( + self.col.build_search_string( + SearchNode(note=self.name), f'"id:{highlight.id}"' + ) + ) + if nids: + note = self.col.get_note(nids[0]) + added = False + else: + note = self.col.new_note(self.notetype) + added = True + note["Question"] = self._format_field(f"{{c1::{highlight.text}}}") + note["Answer"] = self._format_field(highlight.text) + note["id"] = self._format_field(highlight.id) + note["text"] = markdown(self._format_field(highlight.text)) + note["note"] = self._format_field(highlight.note) + note["url"] = self._format_url(highlight.url) + note["readwise_url"] = self._format_url(highlight.readwise_url) + note["highlighted_at"] = self._format_field(highlight.highlighted_at) + note["created_at"] = self._format_field(highlight.created_at) + note["updated_at"] = self._format_field(highlight.updated_at) + note["location"] = self._format_field(highlight.location) + note["end_location"] = self._format_field(highlight.end_location) + note["color"] = self._format_field(highlight.color) + note["is_favorite"] = self._format_field(highlight.is_favorite) + note["is_discard"] = self._format_field(highlight.is_discard) + note["user_book_id"] = self._format_field(doc.user_book_id) + note["readable_title"] = self._format_field(doc.readable_title) + note["title"] = self._format_field(doc.title) + note["document_note"] = self._format_field(doc.document_note) + note["document_readwise_url"] = self._format_url(doc.readwise_url) + note["author"] = self._format_field(doc.author) + note["source"] = self._format_field(doc.source) + note["source_url"] = self._format_url(doc.source_url) + note["unique_url"] = self._format_url(doc.unique_url) + note["cover_image_url"] = self._format_image(doc.cover_image_url) + note["category"] = self._format_field(doc.category) + note["asin"] = self._format_field(doc.asin) + note.tags = [tag["name"] for tag in doc.book_tags + highlight.tags] + + return note, added diff --git a/readwise.py b/readwise.py index 86ce9d2..d2ec47c 100644 --- a/readwise.py +++ b/readwise.py @@ -3,7 +3,8 @@ import logging from dataclasses import dataclass -MODULE_NAME = __name__.split('.')[-1] +MODULE_NAME = __name__.split(".")[-1] + @dataclass class ReadwiseHighlight: @@ -51,11 +52,11 @@ def __post_init__(self): class ReadwiseClient: - def __init__(self, api_key: str=None): + def __init__(self, api_key: str = None): self._base_url = f"https://readwise.io/api/v2" self._parent_logger = None self._logger = logging.getLogger(MODULE_NAME) - self.latest_fetch_time = None + self._latest_fetch_time = None self.set_api_key(api_key) def set_parent_logger(self, parent_logger): @@ -63,29 +64,44 @@ def set_parent_logger(self, parent_logger): self._logger = self._parent_logger.getChild(MODULE_NAME) return self + def set_latest_fetch_time(self, latest_fetch_time): + self._latest_fetch_time = latest_fetch_time + return self + + @property + def latest_fetch_time(self): + return self._latest_fetch_time + def set_api_key(self, api_key): self._api_key = api_key return self - + def _update_time(self): - self.latest_fetch_time = datetime.datetime.now() - + self._latest_fetch_time = datetime.datetime.now() + # Taken from https://readwise.io/api_deets def export(self, updated_after=None): + # TODO: Add support for backing off + # The Readwise API returns a value to backoff for. See https://readwise.io/api_deets self._logger.info("Exporting Readwise data") self._update_time() full_data = [] next_page_cursor = None while True: params = {} - if next_page_cursor: params["pageCursor"] = next_page_cursor - if updated_after: params["updatedAfter"] = updated_after - self._logger.debug(f"Making Readwise export API request with params={params}") + if next_page_cursor: + params["pageCursor"] = next_page_cursor + if updated_after: + params["updatedAfter"] = updated_after + self._logger.debug( + f"Making Readwise export API request with params={params}" + ) response = requests.get( url=f"{self._base_url}/export/", params=params, headers={"Authorization": f"Token {self._api_key}"}, - verify=True) + verify=True, + ) try: response.raise_for_status() json_data = response.json() @@ -97,7 +113,8 @@ def export(self, updated_after=None): full_data.extend(ReadwiseDocument(**d) for d in results) self._logger.debug(f"Fetched {len(results)} documents in this page") next_page_cursor = json_data.get("nextPageCursor") - if not next_page_cursor: break + if not next_page_cursor: + break self._logger.info("Finished exporting Readwise data") self._logger.debug(f"Fetched {len(full_data)} documents in total") num_doc_notes = sum(1 for d in full_data if d.document_note) @@ -105,8 +122,9 @@ def export(self, updated_after=None): num_highlights = sum(len(d.highlights) for d in full_data if d.highlights) self._logger.debug(f"Fetched {num_highlights} highlights in total") return full_data - + def updates(self): - if not self.latest_fetch_time: return self.export() + if not self.latest_fetch_time: + return self.export() self._update_time() return self.export(updated_after=self.latest_fetch_time.isoformat()) diff --git a/vendor/openai/__init__.py b/vendor/openai/__init__.py index 879fe33..2b18422 100644 --- a/vendor/openai/__init__.py +++ b/vendor/openai/__init__.py @@ -7,6 +7,8 @@ from typing import Optional, TYPE_CHECKING from openai.api_resources import ( + Audio, + ChatCompletion, Completion, Customer, Edit, @@ -52,6 +54,8 @@ __all__ = [ "APIError", + "Audio", + "ChatCompletion", "Completion", "Customer", "Edit", @@ -74,7 +78,7 @@ "app_info", "ca_bundle_path", "debug", - "enable_elemetry", + "enable_telemetry", "log", "organization", "proxy", diff --git a/vendor/openai/api_requestor.py b/vendor/openai/api_requestor.py index 8647894..8a0e6ca 100644 --- a/vendor/openai/api_requestor.py +++ b/vendor/openai/api_requestor.py @@ -96,8 +96,10 @@ def parse_stream_helper(line: bytes) -> Optional[str]: # and it will close http connection with TCP Reset return None if line.startswith(b"data: "): - line = line[len(b"data: ") :] - return line.decode("utf-8") + line = line[len(b"data: "):] + return line.decode("utf-8") + else: + return None return None @@ -109,13 +111,10 @@ def parse_stream(rbody: Iterator[bytes]) -> Iterator[str]: async def parse_stream_async(rbody: aiohttp.StreamReader): - async for chunk, _ in rbody.iter_chunks(): - # While the `ChunkTupleAsyncStreamIterator` iterator is meant to iterate over chunks (and thus lines) it seems - # to still sometimes return multiple lines at a time, so let's split the chunk by lines again. - for line in chunk.splitlines(): - _line = parse_stream_helper(line) - if _line is not None: - yield _line + async for line in rbody: + _line = parse_stream_helper(line) + if _line is not None: + yield _line class APIRequestor: @@ -477,8 +476,8 @@ def _prepare_request_raw( abs_url = _build_api_url(abs_url, encoded_params) elif method in {"post", "put"}: if params and files: - raise ValueError("At most one of params and files may be specified.") - if params: + data = params + if params and not files: data = json.dumps(params).encode() headers["Content-Type"] = "application/json" else: @@ -667,7 +666,10 @@ def _interpret_response_line( headers=rheaders, ) try: - data = json.loads(rbody) + if 'text/plain' in rheaders.get('Content-Type'): + data = rbody + else: + data = json.loads(rbody) except (JSONDecodeError, UnicodeDecodeError) as e: raise error.APIError( f"HTTP code {rcode} from API ({rbody})", rbody, rcode, headers=rheaders diff --git a/vendor/openai/api_resources/__init__.py b/vendor/openai/api_resources/__init__.py index 4692b9a..b06ebb4 100644 --- a/vendor/openai/api_resources/__init__.py +++ b/vendor/openai/api_resources/__init__.py @@ -1,3 +1,5 @@ +from openai.api_resources.audio import Audio # noqa: F401 +from openai.api_resources.chat_completion import ChatCompletion # noqa: F401 from openai.api_resources.completion import Completion # noqa: F401 from openai.api_resources.customer import Customer # noqa: F401 from openai.api_resources.deployment import Deployment # noqa: F401 diff --git a/vendor/openai/api_resources/abstract/api_resource.py b/vendor/openai/api_resources/abstract/api_resource.py index 53a7dec..5d54bb9 100644 --- a/vendor/openai/api_resources/abstract/api_resource.py +++ b/vendor/openai/api_resources/abstract/api_resource.py @@ -16,7 +16,7 @@ class APIResource(OpenAIObject): def retrieve( cls, id, api_key=None, request_id=None, request_timeout=None, **params ): - instance = cls(id, api_key, **params) + instance = cls(id=id, api_key=api_key, **params) instance.refresh(request_id=request_id, request_timeout=request_timeout) return instance @@ -24,7 +24,7 @@ def retrieve( def aretrieve( cls, id, api_key=None, request_id=None, request_timeout=None, **params ): - instance = cls(id, api_key, **params) + instance = cls(id=id, api_key=api_key, **params) return instance.arefresh(request_id=request_id, request_timeout=request_timeout) def refresh(self, request_id=None, request_timeout=None): diff --git a/vendor/openai/api_resources/audio.py b/vendor/openai/api_resources/audio.py new file mode 100644 index 0000000..8ad6705 --- /dev/null +++ b/vendor/openai/api_resources/audio.py @@ -0,0 +1,205 @@ +from typing import Any, List + +import openai +from openai import api_requestor, util +from openai.api_resources.abstract import APIResource + + +class Audio(APIResource): + OBJECT_NAME = "audio" + + @classmethod + def _get_url(cls, action): + return cls.class_url() + f"/{action}" + + @classmethod + def _prepare_request( + cls, + file, + filename, + model, + api_key=None, + api_base=None, + api_type=None, + api_version=None, + organization=None, + **params, + ): + requestor = api_requestor.APIRequestor( + api_key, + api_base=api_base or openai.api_base, + api_type=api_type, + api_version=api_version, + organization=organization, + ) + files: List[Any] = [] + data = { + "model": model, + **params, + } + files.append(("file", (filename, file, "application/octet-stream"))) + return requestor, files, data + + @classmethod + def transcribe( + cls, + model, + file, + api_key=None, + api_base=None, + api_type=None, + api_version=None, + organization=None, + **params, + ): + requestor, files, data = cls._prepare_request(file, file.name, model, **params) + url = cls._get_url("transcriptions") + response, _, api_key = requestor.request("post", url, files=files, params=data) + return util.convert_to_openai_object( + response, api_key, api_version, organization + ) + + @classmethod + def translate( + cls, + model, + file, + api_key=None, + api_base=None, + api_type=None, + api_version=None, + organization=None, + **params, + ): + requestor, files, data = cls._prepare_request(file, file.name, model, **params) + url = cls._get_url("translations") + response, _, api_key = requestor.request("post", url, files=files, params=data) + return util.convert_to_openai_object( + response, api_key, api_version, organization + ) + + @classmethod + def transcribe_raw( + cls, + model, + file, + filename, + api_key=None, + api_base=None, + api_type=None, + api_version=None, + organization=None, + **params, + ): + requestor, files, data = cls._prepare_request(file, filename, model, **params) + url = cls._get_url("transcriptions") + response, _, api_key = requestor.request("post", url, files=files, params=data) + return util.convert_to_openai_object( + response, api_key, api_version, organization + ) + + @classmethod + def translate_raw( + cls, + model, + file, + filename, + api_key=None, + api_base=None, + api_type=None, + api_version=None, + organization=None, + **params, + ): + requestor, files, data = cls._prepare_request(file, filename, model, **params) + url = cls._get_url("translations") + response, _, api_key = requestor.request("post", url, files=files, params=data) + return util.convert_to_openai_object( + response, api_key, api_version, organization + ) + + @classmethod + async def atranscribe( + cls, + model, + file, + api_key=None, + api_base=None, + api_type=None, + api_version=None, + organization=None, + **params, + ): + requestor, files, data = cls._prepare_request(file, file.name, model, **params) + url = cls._get_url("transcriptions") + response, _, api_key = await requestor.arequest( + "post", url, files=files, params=data + ) + return util.convert_to_openai_object( + response, api_key, api_version, organization + ) + + @classmethod + async def atranslate( + cls, + model, + file, + api_key=None, + api_base=None, + api_type=None, + api_version=None, + organization=None, + **params, + ): + requestor, files, data = cls._prepare_request(file, file.name, model, **params) + url = cls._get_url("translations") + response, _, api_key = await requestor.arequest( + "post", url, files=files, params=data + ) + return util.convert_to_openai_object( + response, api_key, api_version, organization + ) + + @classmethod + async def atranscribe_raw( + cls, + model, + file, + filename, + api_key=None, + api_base=None, + api_type=None, + api_version=None, + organization=None, + **params, + ): + requestor, files, data = cls._prepare_request(file, filename, model, **params) + url = cls._get_url("transcriptions") + response, _, api_key = await requestor.arequest( + "post", url, files=files, params=data + ) + return util.convert_to_openai_object( + response, api_key, api_version, organization + ) + + @classmethod + async def atranslate_raw( + cls, + model, + file, + filename, + api_key=None, + api_base=None, + api_type=None, + api_version=None, + organization=None, + **params, + ): + requestor, files, data = cls._prepare_request(file, filename, model, **params) + url = cls._get_url("translations") + response, _, api_key = await requestor.arequest( + "post", url, files=files, params=data + ) + return util.convert_to_openai_object( + response, api_key, api_version, organization + ) diff --git a/vendor/openai/api_resources/chat_completion.py b/vendor/openai/api_resources/chat_completion.py new file mode 100644 index 0000000..39fb58b --- /dev/null +++ b/vendor/openai/api_resources/chat_completion.py @@ -0,0 +1,50 @@ +import time + +from openai import util +from openai.api_resources.abstract.engine_api_resource import EngineAPIResource +from openai.error import TryAgain + + +class ChatCompletion(EngineAPIResource): + engine_required = False + OBJECT_NAME = "chat.completions" + + @classmethod + def create(cls, *args, **kwargs): + """ + Creates a new chat completion for the provided messages and parameters. + + See https://platform.openai.com/docs/api-reference/chat-completions/create + for a list of valid parameters. + """ + start = time.time() + timeout = kwargs.pop("timeout", None) + + while True: + try: + return super().create(*args, **kwargs) + except TryAgain as e: + if timeout is not None and time.time() > start + timeout: + raise + + util.log_info("Waiting for model to warm up", error=e) + + @classmethod + async def acreate(cls, *args, **kwargs): + """ + Creates a new chat completion for the provided messages and parameters. + + See https://platform.openai.com/docs/api-reference/chat-completions/create + for a list of valid parameters. + """ + start = time.time() + timeout = kwargs.pop("timeout", None) + + while True: + try: + return await super().acreate(*args, **kwargs) + except TryAgain as e: + if timeout is not None and time.time() > start + timeout: + raise + + util.log_info("Waiting for model to warm up", error=e) diff --git a/vendor/openai/api_resources/completion.py b/vendor/openai/api_resources/completion.py index 6912b4b..7b9c44b 100644 --- a/vendor/openai/api_resources/completion.py +++ b/vendor/openai/api_resources/completion.py @@ -14,7 +14,7 @@ def create(cls, *args, **kwargs): """ Creates a new completion for the provided prompt and parameters. - See https://beta.openai.com/docs/api-reference/completions/create for a list + See https://platform.openai.com/docs/api-reference/completions/create for a list of valid parameters. """ start = time.time() @@ -34,7 +34,7 @@ async def acreate(cls, *args, **kwargs): """ Creates a new completion for the provided prompt and parameters. - See https://beta.openai.com/docs/api-reference/completions/create for a list + See https://platform.openai.com/docs/api-reference/completions/create for a list of valid parameters. """ start = time.time() diff --git a/vendor/openai/api_resources/customer.py b/vendor/openai/api_resources/customer.py index cb9779a..8690d07 100644 --- a/vendor/openai/api_resources/customer.py +++ b/vendor/openai/api_resources/customer.py @@ -3,7 +3,7 @@ class Customer(OpenAIObject): @classmethod - def get_url(self, customer, endpoint): + def get_url(cls, customer, endpoint): return f"/customer/{customer}/{endpoint}" @classmethod diff --git a/vendor/openai/api_resources/embedding.py b/vendor/openai/api_resources/embedding.py index 5f1cfe5..4eb97c6 100644 --- a/vendor/openai/api_resources/embedding.py +++ b/vendor/openai/api_resources/embedding.py @@ -16,7 +16,7 @@ def create(cls, *args, **kwargs): """ Creates a new embedding for the provided input and parameters. - See https://beta.openai.com/docs/api-reference/embeddings for a list + See https://platform.openai.com/docs/api-reference/embeddings for a list of valid parameters. """ start = time.time() @@ -56,7 +56,7 @@ async def acreate(cls, *args, **kwargs): """ Creates a new embedding for the provided input and parameters. - See https://beta.openai.com/docs/api-reference/embeddings for a list + See https://platform.openai.com/docs/api-reference/embeddings for a list of valid parameters. """ start = time.time() diff --git a/vendor/openai/api_resources/file.py b/vendor/openai/api_resources/file.py index 365cb2a..3944172 100644 --- a/vendor/openai/api_resources/file.py +++ b/vendor/openai/api_resources/file.py @@ -198,7 +198,7 @@ async def adownload( return result.content @classmethod - def __find_matching_files(cls, name, all_files, purpose): + def __find_matching_files(cls, name, bytes, all_files, purpose): matching_files = [] basename = os.path.basename(name) for f in all_files: @@ -234,7 +234,7 @@ def find_matching_files( api_version=api_version, organization=organization, ).get("data", []) - return cls.__find_matching_files(name, all_files, purpose) + return cls.__find_matching_files(name, bytes, all_files, purpose) @classmethod async def afind_matching_files( @@ -258,4 +258,4 @@ async def afind_matching_files( organization=organization, ) ).get("data", []) - return cls.__find_matching_files(name, all_files, purpose) + return cls.__find_matching_files(name, bytes, all_files, purpose) diff --git a/vendor/openai/api_resources/moderation.py b/vendor/openai/api_resources/moderation.py index 4b8b58c..bd19646 100644 --- a/vendor/openai/api_resources/moderation.py +++ b/vendor/openai/api_resources/moderation.py @@ -7,7 +7,7 @@ class Moderation(OpenAIObject): VALID_MODEL_NAMES: List[str] = ["text-moderation-stable", "text-moderation-latest"] @classmethod - def get_url(self): + def get_url(cls): return "/moderations" @classmethod diff --git a/vendor/openai/cli.py b/vendor/openai/cli.py index a7f7654..e9201b1 100644 --- a/vendor/openai/cli.py +++ b/vendor/openai/cli.py @@ -108,6 +108,44 @@ def list(cls, args): display(engines) +class ChatCompletion: + @classmethod + def create(cls, args): + if args.n is not None and args.n > 1 and args.stream: + raise ValueError( + "Can't stream chat completions with n>1 with the current CLI" + ) + + messages = [ + {"role": role, "content": content} for role, content in args.message + ] + + resp = openai.ChatCompletion.create( + # Required + model=args.model, + messages=messages, + # Optional + n=args.n, + max_tokens=100, + temperature=args.temperature, + top_p=args.top_p, + stop=args.stop, + stream=args.stream, + ) + if not args.stream: + resp = [resp] + + for part in resp: + choices = part["choices"] + for c_idx, c in enumerate(sorted(choices, key=lambda s: s["index"])): + if len(choices) > 1: + sys.stdout.write("===== Chat Completion {} =====\n".format(c_idx)) + sys.stdout.write(c["message"]["content"]) + if len(choices) > 1: + sys.stdout.write("\n") + sys.stdout.flush() + + class Completion: @classmethod def create(cls, args): @@ -255,6 +293,43 @@ def create_edit(cls, args): print(resp) +class Audio: + @classmethod + def transcribe(cls, args): + with open(args.file, "rb") as r: + file_reader = BufferReader(r.read(), desc="Upload progress") + + resp = openai.Audio.transcribe_raw( + # Required + model=args.model, + file=file_reader, + filename=args.file, + # Optional + response_format=args.response_format, + language=args.language, + temperature=args.temperature, + prompt=args.prompt, + ) + print(resp) + + @classmethod + def translate(cls, args): + with open(args.file, "rb") as r: + file_reader = BufferReader(r.read(), desc="Upload progress") + resp = openai.Audio.translate_raw( + # Required + model=args.model, + file=file_reader, + filename=args.file, + # Optional + response_format=args.response_format, + language=args.language, + temperature=args.temperature, + prompt=args.prompt, + ) + print(resp) + + class FineTune: @classmethod def list(cls, args): @@ -505,7 +580,6 @@ def delete(cls, args): @classmethod def prepare_data(cls, args): - sys.stdout.write("Analyzing...\n") fname = args.file auto_accept = args.quiet @@ -633,12 +707,68 @@ def help(args): ) sub.set_defaults(func=Engine.generate) + # Chat Completions + sub = subparsers.add_parser("chat_completions.create") + + sub._action_groups.pop() + req = sub.add_argument_group("required arguments") + opt = sub.add_argument_group("optional arguments") + + req.add_argument( + "-m", + "--model", + help="The model to use.", + required=True, + ) + req.add_argument( + "-g", + "--message", + action="append", + nargs=2, + metavar=("ROLE", "CONTENT"), + help="A message in `{role} {content}` format. Use this argument multiple times to add multiple messages.", + required=True, + ) + opt.add_argument( + "-n", + "--n", + help="How many completions to generate for the conversation.", + type=int, + ) + opt.add_argument( + "-M", "--max-tokens", help="The maximum number of tokens to generate.", type=int + ) + opt.add_argument( + "-t", + "--temperature", + help="""What sampling temperature to use. Higher values means the model will take more risks. Try 0.9 for more creative applications, and 0 (argmax sampling) for ones with a well-defined answer. + +Mutually exclusive with `top_p`.""", + type=float, + ) + opt.add_argument( + "-P", + "--top_p", + help="""An alternative to sampling with temperature, called nucleus sampling, where the considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10%% probability mass are considered. + + Mutually exclusive with `temperature`.""", + type=float, + ) + opt.add_argument( + "--stop", + help="A stop sequence at which to stop generating tokens for the message.", + ) + opt.add_argument( + "--stream", help="Stream messages as they're ready.", action="store_true" + ) + sub.set_defaults(func=ChatCompletion.create) + # Completions sub = subparsers.add_parser("completions.create") sub.add_argument( "-e", "--engine", - help="The engine to use. See https://beta.openai.com/docs/engines for more about what engines are available.", + help="The engine to use. See https://platform.openai.com/docs/engines for more about what engines are available.", ) sub.add_argument( "-m", @@ -725,7 +855,7 @@ def help(args): sub.add_argument( "-p", "--purpose", - help="Why are you uploading this file? (see https://beta.openai.com/docs/api-reference/ for purposes)", + help="Why are you uploading this file? (see https://platform.openai.com/docs/api-reference/ for purposes)", required=True, ) sub.set_defaults(func=File.create) @@ -924,6 +1054,30 @@ def help(args): sub.add_argument("--response-format", type=str, default="url") sub.set_defaults(func=Image.create_variation) + # Audio + # transcriptions + sub = subparsers.add_parser("audio.transcribe") + # Required + sub.add_argument("-m", "--model", type=str, default="whisper-1") + sub.add_argument("-f", "--file", type=str, required=True) + # Optional + sub.add_argument("--response-format", type=str) + sub.add_argument("--language", type=str) + sub.add_argument("-t", "--temperature", type=float) + sub.add_argument("--prompt", type=str) + sub.set_defaults(func=Audio.transcribe) + # translations + sub = subparsers.add_parser("audio.translate") + # Required + sub.add_argument("-m", "--model", type=str, default="whisper-1") + sub.add_argument("-f", "--file", type=str, required=True) + # Optional + sub.add_argument("--response-format", type=str) + sub.add_argument("--language", type=str) + sub.add_argument("-t", "--temperature", type=float) + sub.add_argument("--prompt", type=str) + sub.set_defaults(func=Audio.translate) + def wandb_register(parser): subparsers = parser.add_subparsers( diff --git a/vendor/openai/tests/asyncio/test_endpoints.py b/vendor/openai/tests/asyncio/test_endpoints.py index 3dc355b..1b146e6 100644 --- a/vendor/openai/tests/asyncio/test_endpoints.py +++ b/vendor/openai/tests/asyncio/test_endpoints.py @@ -7,17 +7,18 @@ import openai from openai import error - pytestmark = [pytest.mark.asyncio] # FILE TESTS async def test_file_upload(): result = await openai.File.acreate( - file=io.StringIO(json.dumps({"text": "test file data"})), - purpose="search", + file=io.StringIO( + json.dumps({"prompt": "test file data", "completion": "tada"}) + ), + purpose="fine-tune", ) - assert result.purpose == "search" + assert result.purpose == "fine-tune" assert "id" in result result = await openai.File.aretrieve(id=result.id) diff --git a/vendor/openai/tests/test_endpoints.py b/vendor/openai/tests/test_endpoints.py index 565bd41..c3fc109 100644 --- a/vendor/openai/tests/test_endpoints.py +++ b/vendor/openai/tests/test_endpoints.py @@ -22,6 +22,32 @@ def test_file_upload(): assert result.status == "uploaded" +# CHAT COMPLETION TESTS +def test_chat_completions(): + result = openai.ChatCompletion.create( + model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hello!"}] + ) + assert len(result.choices) == 1 + + +def test_chat_completions_multiple(): + result = openai.ChatCompletion.create( + model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hello!"}], n=5 + ) + assert len(result.choices) == 5 + + +def test_chat_completions_streaming(): + result = None + events = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hello!"}], + stream=True, + ) + for result in events: + assert len(result.choices) == 1 + + # COMPLETION TESTS def test_completions(): result = openai.Completion.create(prompt="This was a test", n=5, engine="ada") diff --git a/vendor/openai/tests/test_long_examples_validator.py b/vendor/openai/tests/test_long_examples_validator.py index a9334d4..0cac136 100644 --- a/vendor/openai/tests/test_long_examples_validator.py +++ b/vendor/openai/tests/test_long_examples_validator.py @@ -4,7 +4,12 @@ import pytest -from openai.datalib import HAS_PANDAS, HAS_NUMPY, NUMPY_INSTRUCTIONS, PANDAS_INSTRUCTIONS +from openai.datalib import ( + HAS_NUMPY, + HAS_PANDAS, + NUMPY_INSTRUCTIONS, + PANDAS_INSTRUCTIONS, +) @pytest.mark.skipif(not HAS_PANDAS, reason=PANDAS_INSTRUCTIONS) @@ -29,7 +34,8 @@ def test_long_examples_validator() -> None: {"prompt": long_prompt, "completion": long_completion}, # 2 of 2 duplicates ] - with NamedTemporaryFile(suffix="jsonl", mode="w") as training_data: + with NamedTemporaryFile(suffix=".jsonl", mode="w") as training_data: + print(training_data.name) for prompt_completion_row in unprepared_training_data: training_data.write(json.dumps(prompt_completion_row) + "\n") training_data.flush() diff --git a/vendor/openai/validators.py b/vendor/openai/validators.py index c5a3dd7..b15e59b 100644 --- a/vendor/openai/validators.py +++ b/vendor/openai/validators.py @@ -242,7 +242,7 @@ def add_suffix(x, suffix): immediate_msg += f"\n WARNING: Some of your prompts contain the suffix `{common_suffix}` more than once. We strongly suggest that you review your prompts and add a unique suffix" else: - immediate_msg = "\n- Your data does not contain a common separator at the end of your prompts. Having a separator string appended to the end of the prompt makes it clearer to the fine-tuned model where the completion should begin. See https://beta.openai.com/docs/guides/fine-tuning/preparing-your-dataset for more detail and examples. If you intend to do open-ended generation, then you should leave the prompts empty" + immediate_msg = "\n- Your data does not contain a common separator at the end of your prompts. Having a separator string appended to the end of the prompt makes it clearer to the fine-tuned model where the completion should begin. See https://platform.openai.com/docs/guides/fine-tuning/preparing-your-dataset for more detail and examples. If you intend to do open-ended generation, then you should leave the prompts empty" if common_suffix == "": optional_msg = ( @@ -393,7 +393,7 @@ def add_suffix(x, suffix): immediate_msg += f"\n WARNING: Some of your completions contain the suffix `{common_suffix}` more than once. We suggest that you review your completions and add a unique ending" else: - immediate_msg = "\n- Your data does not contain a common ending at the end of your completions. Having a common ending string appended to the end of the completion makes it clearer to the fine-tuned model where the completion should end. See https://beta.openai.com/docs/guides/fine-tuning/preparing-your-dataset for more detail and examples." + immediate_msg = "\n- Your data does not contain a common ending at the end of your completions. Having a common ending string appended to the end of the completion makes it clearer to the fine-tuned model where the completion should end. See https://platform.openai.com/docs/guides/fine-tuning/preparing-your-dataset for more detail and examples." if common_suffix == "": optional_msg = ( @@ -428,7 +428,7 @@ def add_space_start(x): immediate_msg = None if df.completion.str[:1].nunique() != 1 or df.completion.values[0][0] != " ": - immediate_msg = "\n- The completion should start with a whitespace character (` `). This tends to produce better results due to the tokenization we use. See https://beta.openai.com/docs/guides/fine-tuning/preparing-your-dataset for more details" + immediate_msg = "\n- The completion should start with a whitespace character (` `). This tends to produce better results due to the tokenization we use. See https://platform.openai.com/docs/guides/fine-tuning/preparing-your-dataset for more details" optional_msg = "Add a whitespace character to the beginning of the completion" optional_fn = add_space_start return Remediation( @@ -462,7 +462,7 @@ def lower_case(x): if count_upper * 2 > count_lower: return Remediation( name="lower_case", - immediate_msg=f"\n- More than a third of your `{column}` column/key is uppercase. Uppercase {column}s tends to perform worse than a mixture of case encountered in normal language. We recommend to lower case the data if that makes sense in your domain. See https://beta.openai.com/docs/guides/fine-tuning/preparing-your-dataset for more details", + immediate_msg=f"\n- More than a third of your `{column}` column/key is uppercase. Uppercase {column}s tends to perform worse than a mixture of case encountered in normal language. We recommend to lower case the data if that makes sense in your domain. See https://platform.openai.com/docs/guides/fine-tuning/preparing-your-dataset for more details", optional_msg=f"Lowercase all your data in column/key `{column}`", optional_fn=lower_case, ) diff --git a/vendor/openai/version.py b/vendor/openai/version.py index cbd0c64..cba8d89 100644 --- a/vendor/openai/version.py +++ b/vendor/openai/version.py @@ -1 +1 @@ -VERSION = "0.26.4" +VERSION = "0.27.2"