From 68df99a04ada88551974ae4496acb4efe16a33f7 Mon Sep 17 00:00:00 2001 From: Abhi Somala Date: Mon, 8 Jul 2024 14:07:53 -0400 Subject: [PATCH 01/12] added connector folder and HF file --- connectors/huggingface_connecter.py | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 connectors/huggingface_connecter.py diff --git a/connectors/huggingface_connecter.py b/connectors/huggingface_connecter.py new file mode 100644 index 00000000..e39d7562 --- /dev/null +++ b/connectors/huggingface_connecter.py @@ -0,0 +1,46 @@ +from datasets import load_dataset +import hashlib + +# Gets data from a Hugging Face dataset with automatic configuration +def fetch_data_from_huggingface(dataset_identifier): + try: + # Try loading the dataset without specifying a configuration + dataset = load_dataset(dataset_identifier, trust_remote_code=True) + except ValueError as e: + # If there is an error it might be because of the config selection + if "Please pick one among the available configs" in str(e): + # Gets available config and selects first one + available_configs = str(e).split("['")[1].split("']")[0].split("', '") + dataset = load_dataset(dataset_identifier, available_configs[0], trust_remote_code=True) + else: + raise e + + data = [] + for split in dataset.keys(): + for i, example in enumerate(dataset[split]): + # Creates a unique and shortened ID + unique_str = f"{dataset_identifier}_{split}_{i}" + short_id = hashlib.sha1(unique_str.encode()).hexdigest()[:25] + example['id'] = short_id + data.append(example) + return data + +# Main load function to be used as a connector +def load(dataset_identifier): + data = fetch_data_from_huggingface(dataset_identifier.strip()) + + if data: + return data + else: + raise ValueError("No data was found for the provided dataset.") +if __name__ == "__main__": + dataset_identifier = input("Enter Hugging Face dataset identifier: ").strip() + + try: + data = load(dataset_identifier) + print(f"Dataset has been loaded successfully. Number of entries: {len(data)}") + # You can add more processing logic here if needed + except ValueError as e: + print(f"Error loading dataset: {e}") + + From 16257b9865e47cd41a1dfb71c04b1a1eab4c4e74 Mon Sep 17 00:00:00 2001 From: abhisomala <68791501+abhisomala@users.noreply.github.com> Date: Mon, 8 Jul 2024 14:19:56 -0400 Subject: [PATCH 02/12] Fixed comments from ellipsis-dev bot --- connectors/huggingface_connecter.py | 35 +++++++++++++++++++---------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/connectors/huggingface_connecter.py b/connectors/huggingface_connecter.py index e39d7562..118d8d6f 100644 --- a/connectors/huggingface_connecter.py +++ b/connectors/huggingface_connecter.py @@ -1,28 +1,37 @@ from datasets import load_dataset import hashlib -# Gets data from a Hugging Face dataset with automatic configuration +# Fetches data from a Hugging Face dataset def fetch_data_from_huggingface(dataset_identifier): try: - # Try loading the dataset without specifying a configuration - dataset = load_dataset(dataset_identifier, trust_remote_code=True) + # Attempts to load dataset without specifying config + dataset = load_dataset(dataset_identifier) except ValueError as e: - # If there is an error it might be because of the config selection - if "Please pick one among the available configs" in str(e): - # Gets available config and selects first one - available_configs = str(e).split("['")[1].split("']")[0].split("', '") - dataset = load_dataset(dataset_identifier, available_configs[0], trust_remote_code=True) + # Handles error messages + error_message = str(e) + if "Please pick one among the available configs" in error_message: + + try: + available_configs_start = error_message.index("['") + 2 + available_configs_end = error_message.index("']") + available_configs = error_message[available_configs_start:available_configs_end].split("', '") + # Load dataset with the first available config + dataset = load_dataset(dataset_identifier, available_configs[0], trust_remote_code=True) + except ValueError: + raise ValueError("Failed to extract available configurations from the error message.") else: - raise e + raise e + # Processes dataset entries data = [] for split in dataset.keys(): for i, example in enumerate(dataset[split]): - # Creates a unique and shortened ID + # Creates a unique ID using SHA-256 for better security unique_str = f"{dataset_identifier}_{split}_{i}" - short_id = hashlib.sha1(unique_str.encode()).hexdigest()[:25] + short_id = hashlib.sha256(unique_str.encode()).hexdigest()[:25] example['id'] = short_id data.append(example) + return data # Main load function to be used as a connector @@ -33,14 +42,16 @@ def load(dataset_identifier): return data else: raise ValueError("No data was found for the provided dataset.") + if __name__ == "__main__": dataset_identifier = input("Enter Hugging Face dataset identifier: ").strip() try: data = load(dataset_identifier) print(f"Dataset has been loaded successfully. Number of entries: {len(data)}") - # You can add more processing logic here if needed + except ValueError as e: print(f"Error loading dataset: {e}") + From 4cd48ff8e3b5b0311e9340ef754d0dd0944a70c3 Mon Sep 17 00:00:00 2001 From: Abhi Somala Date: Mon, 8 Jul 2024 16:33:37 -0400 Subject: [PATCH 03/12] added init.py and edits to HF connecter --- connectors/__init__.py | 5 ++ connectors/huggingface_connecter.py | 80 ++++++++++++++++++----------- 2 files changed, 55 insertions(+), 30 deletions(-) create mode 100644 connectors/__init__.py diff --git a/connectors/__init__.py b/connectors/__init__.py new file mode 100644 index 00000000..1ce04123 --- /dev/null +++ b/connectors/__init__.py @@ -0,0 +1,5 @@ +from nomic.connectors import huggingface_connecter + +atlas_dataset = huggingface_connecter.load('aaa/bbb') + +atlas_dataset.create_index(...) \ No newline at end of file diff --git a/connectors/huggingface_connecter.py b/connectors/huggingface_connecter.py index e39d7562..0c07f225 100644 --- a/connectors/huggingface_connecter.py +++ b/connectors/huggingface_connecter.py @@ -1,46 +1,66 @@ from datasets import load_dataset -import hashlib +from ulid import ULID -# Gets data from a Hugging Face dataset with automatic configuration -def fetch_data_from_huggingface(dataset_identifier): + +#need to add an init.py + +# Function to fetch data from a Hugging Face dataset +def fetch_data_from_huggingface(dataset_identifier, dataset_split=None): try: - # Try loading the dataset without specifying a configuration - dataset = load_dataset(dataset_identifier, trust_remote_code=True) - except ValueError as e: - # If there is an error it might be because of the config selection - if "Please pick one among the available configs" in str(e): - # Gets available config and selects first one - available_configs = str(e).split("['")[1].split("']")[0].split("', '") - dataset = load_dataset(dataset_identifier, available_configs[0], trust_remote_code=True) + # Attempt to load the dataset without specifying a configuration + dataset = load_dataset(dataset_identifier) + + if dataset_split is None: + # Use the first available split by default + split = next(iter(dataset.keys())) else: - raise e - - data = [] - for split in dataset.keys(): + # Use the specified split + split = dataset_split + + data = [] + ############ for i, example in enumerate(dataset[split]): - # Creates a unique and shortened ID - unique_str = f"{dataset_identifier}_{split}_{i}" - short_id = hashlib.sha1(unique_str.encode()).hexdigest()[:25] - example['id'] = short_id + # Create a unique ULID + ulid = ULID() + example['id'] = str(ulid) data.append(example) - return data + + return data + + except ValueError as e: + # Handle error messages + error_message = str(e) + if "Please pick one among the available configs" in error_message: + try: + available_configs_start = error_message.index("['") + 2 + available_configs_end = error_message.index("']") + available_configs = error_message[available_configs_start:available_configs_end].split("', '") + # Load dataset with the first available config + dataset = load_dataset(dataset_identifier, available_configs[0], trust_remote_code=True) + # Proceed with data loading as above + split = next(iter(dataset.keys())) + data = [] + for example in enumerate(dataset[split]): + ulid = ULID() + example['id'] = str(ulid) + data.append(example) + + return data + + except ValueError: + raise ValueError("Failed to extract available configurations from the error message.") + + else: + raise e # Re-raise other ValueErrors # Main load function to be used as a connector -def load(dataset_identifier): - data = fetch_data_from_huggingface(dataset_identifier.strip()) +def load(dataset_identifier, dataset_split=None): + data = fetch_data_from_huggingface(dataset_identifier.strip(), dataset_split) if data: return data else: raise ValueError("No data was found for the provided dataset.") -if __name__ == "__main__": - dataset_identifier = input("Enter Hugging Face dataset identifier: ").strip() - try: - data = load(dataset_identifier) - print(f"Dataset has been loaded successfully. Number of entries: {len(data)}") - # You can add more processing logic here if needed - except ValueError as e: - print(f"Error loading dataset: {e}") From 86b230f05bb57733f4af6a9cdf3e02ef82a3e73a Mon Sep 17 00:00:00 2001 From: Abhi Somala Date: Mon, 8 Jul 2024 16:41:38 -0400 Subject: [PATCH 04/12] minor edits --- connectors/huggingface_connecter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connectors/huggingface_connecter.py b/connectors/huggingface_connecter.py index 0c07f225..abf09d06 100644 --- a/connectors/huggingface_connecter.py +++ b/connectors/huggingface_connecter.py @@ -18,7 +18,7 @@ def fetch_data_from_huggingface(dataset_identifier, dataset_split=None): split = dataset_split data = [] - ############ + for i, example in enumerate(dataset[split]): # Create a unique ULID ulid = ULID() From 497e04bdf1cc049a2522830659fd5bfa1c11636d Mon Sep 17 00:00:00 2001 From: abhisomala <68791501+abhisomala@users.noreply.github.com> Date: Mon, 8 Jul 2024 16:45:58 -0400 Subject: [PATCH 05/12] Made a couple edits (working on init.py) --- connectors/huggingface_connecter.py | 70 ++++++++++++++++------------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/connectors/huggingface_connecter.py b/connectors/huggingface_connecter.py index 118d8d6f..7b75eb54 100644 --- a/connectors/huggingface_connecter.py +++ b/connectors/huggingface_connecter.py @@ -1,57 +1,67 @@ from datasets import load_dataset -import hashlib +from ulid import ULID -# Fetches data from a Hugging Face dataset -def fetch_data_from_huggingface(dataset_identifier): + +#need to add an init.py + +# Function to fetch data from a Hugging Face dataset +def fetch_data_from_huggingface(dataset_identifier, dataset_split=None): try: - # Attempts to load dataset without specifying config + # Attempt to load the dataset without specifying a configuration dataset = load_dataset(dataset_identifier) + + if dataset_split is None: + # Use the first available split by default + split = next(iter(dataset.keys())) + else: + # Use the specified split + split = dataset_split + + data = [] + + for i, example in enumerate(dataset[split]): + # Create a unique ULID + ulid = ULID() + example['id'] = str(ulid) + data.append(example) + + return data + except ValueError as e: - # Handles error messages + # Handle error messages error_message = str(e) if "Please pick one among the available configs" in error_message: - try: available_configs_start = error_message.index("['") + 2 available_configs_end = error_message.index("']") available_configs = error_message[available_configs_start:available_configs_end].split("', '") # Load dataset with the first available config dataset = load_dataset(dataset_identifier, available_configs[0], trust_remote_code=True) + # Proceed with data loading as above + split = next(iter(dataset.keys())) + data = [] + for example in enumerate(dataset[split]): + ulid = ULID() + example['id'] = str(ulid) + data.append(example) + + return data + except ValueError: raise ValueError("Failed to extract available configurations from the error message.") + else: - raise e - - # Processes dataset entries - data = [] - for split in dataset.keys(): - for i, example in enumerate(dataset[split]): - # Creates a unique ID using SHA-256 for better security - unique_str = f"{dataset_identifier}_{split}_{i}" - short_id = hashlib.sha256(unique_str.encode()).hexdigest()[:25] - example['id'] = short_id - data.append(example) - - return data + raise e # Re-raise other ValueErrors # Main load function to be used as a connector -def load(dataset_identifier): - data = fetch_data_from_huggingface(dataset_identifier.strip()) +def load(dataset_identifier, dataset_split=None): + data = fetch_data_from_huggingface(dataset_identifier.strip(), dataset_split) if data: return data else: raise ValueError("No data was found for the provided dataset.") -if __name__ == "__main__": - dataset_identifier = input("Enter Hugging Face dataset identifier: ").strip() - - try: - data = load(dataset_identifier) - print(f"Dataset has been loaded successfully. Number of entries: {len(data)}") - - except ValueError as e: - print(f"Error loading dataset: {e}") From 152a99ed7e008ed8008bb76af21bf094863e7ffe Mon Sep 17 00:00:00 2001 From: Abhi Somala Date: Mon, 8 Jul 2024 22:14:30 -0400 Subject: [PATCH 06/12] updated init.py and created a file for an example --- connectors/__init__.py | 4 +++- connectors/example_usage.py | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 connectors/example_usage.py diff --git a/connectors/__init__.py b/connectors/__init__.py index 1ce04123..a7f70e4e 100644 --- a/connectors/__init__.py +++ b/connectors/__init__.py @@ -2,4 +2,6 @@ atlas_dataset = huggingface_connecter.load('aaa/bbb') -atlas_dataset.create_index(...) \ No newline at end of file +atlas_dataset.create_index(topic_model=True, embedding_model='NomicEmbed') + +print("Atlas dataset has been loaded and indexed successfully.") diff --git a/connectors/example_usage.py b/connectors/example_usage.py new file mode 100644 index 00000000..7a430724 --- /dev/null +++ b/connectors/example_usage.py @@ -0,0 +1,9 @@ +from nomic.connectors import huggingface_connecter + +# Example source url: https://huggingface.co/datasets/allenai/quartz +#Takes last two parts of url to get allenai/quartz +atlas_dataset = huggingface_connecter.load('allenai/quartz') + +atlas_dataset.create_index(topic_model=True, embedding_model='NomicEmbed') + +print("Atlas dataset has been loaded and indexed successfully.") From 2f783ca1fc497207daeba9b2af299549c1a594f1 Mon Sep 17 00:00:00 2001 From: Abhi Somala Date: Tue, 9 Jul 2024 11:53:34 -0400 Subject: [PATCH 07/12] emptied init file,moved HF example usage and changed print statment to log --- connectors/__init__.py | 7 ------- connectors/huggingface_connecter.py | 4 ++-- .../example_usage.py => examples/HF_example_usage.py | 3 ++- 3 files changed, 4 insertions(+), 10 deletions(-) rename connectors/example_usage.py => examples/HF_example_usage.py (78%) diff --git a/connectors/__init__.py b/connectors/__init__.py index a7f70e4e..e69de29b 100644 --- a/connectors/__init__.py +++ b/connectors/__init__.py @@ -1,7 +0,0 @@ -from nomic.connectors import huggingface_connecter - -atlas_dataset = huggingface_connecter.load('aaa/bbb') - -atlas_dataset.create_index(topic_model=True, embedding_model='NomicEmbed') - -print("Atlas dataset has been loaded and indexed successfully.") diff --git a/connectors/huggingface_connecter.py b/connectors/huggingface_connecter.py index 7b75eb54..a3a57255 100644 --- a/connectors/huggingface_connecter.py +++ b/connectors/huggingface_connecter.py @@ -2,7 +2,7 @@ from ulid import ULID -#need to add an init.py + # Function to fetch data from a Hugging Face dataset def fetch_data_from_huggingface(dataset_identifier, dataset_split=None): @@ -53,7 +53,7 @@ def fetch_data_from_huggingface(dataset_identifier, dataset_split=None): else: raise e # Re-raise other ValueErrors -# Main load function to be used as a connector +# Load function to be used as a connector def load(dataset_identifier, dataset_split=None): data = fetch_data_from_huggingface(dataset_identifier.strip(), dataset_split) diff --git a/connectors/example_usage.py b/examples/HF_example_usage.py similarity index 78% rename from connectors/example_usage.py rename to examples/HF_example_usage.py index 7a430724..6078b779 100644 --- a/connectors/example_usage.py +++ b/examples/HF_example_usage.py @@ -1,4 +1,5 @@ from nomic.connectors import huggingface_connecter +import logging # Example source url: https://huggingface.co/datasets/allenai/quartz #Takes last two parts of url to get allenai/quartz @@ -6,4 +7,4 @@ atlas_dataset.create_index(topic_model=True, embedding_model='NomicEmbed') -print("Atlas dataset has been loaded and indexed successfully.") +logging.info("Atlas dataset has been loaded and indexed successfully.") From 9f0575b0d7bd1ff10072688c8fe864a76304166f Mon Sep 17 00:00:00 2001 From: Abhi Somala Date: Tue, 9 Jul 2024 15:44:44 -0400 Subject: [PATCH 08/12] updated connector and example --- connectors/huggingface_connecter.py | 102 +++++++++++++++------------- examples/HF_example_usage.py | 26 +++++-- 2 files changed, 75 insertions(+), 53 deletions(-) diff --git a/connectors/huggingface_connecter.py b/connectors/huggingface_connecter.py index a3a57255..03d9afc5 100644 --- a/connectors/huggingface_connecter.py +++ b/connectors/huggingface_connecter.py @@ -1,67 +1,77 @@ from datasets import load_dataset -from ulid import ULID +from nomic import AtlasDataset +from ulid import ULID +import numpy as np - - - -# Function to fetch data from a Hugging Face dataset -def fetch_data_from_huggingface(dataset_identifier, dataset_split=None): +# Gets data from HF dataset +def get_hfdata(dataset_identifier): try: - # Attempt to load the dataset without specifying a configuration + # Loads dataset without specifying config dataset = load_dataset(dataset_identifier) - - if dataset_split is None: - # Use the first available split by default - split = next(iter(dataset.keys())) - else: - # Use the specified split - split = dataset_split - - data = [] - - for i, example in enumerate(dataset[split]): - # Create a unique ULID - ulid = ULID() - example['id'] = str(ulid) - data.append(example) - - return data - except ValueError as e: - # Handle error messages + # Handles error messages error_message = str(e) if "Please pick one among the available configs" in error_message: try: available_configs_start = error_message.index("['") + 2 available_configs_end = error_message.index("']") available_configs = error_message[available_configs_start:available_configs_end].split("', '") - # Load dataset with the first available config dataset = load_dataset(dataset_identifier, available_configs[0], trust_remote_code=True) - # Proceed with data loading as above - split = next(iter(dataset.keys())) - data = [] - for example in enumerate(dataset[split]): - ulid = ULID() - example['id'] = str(ulid) - data.append(example) - - return data - except ValueError: - raise ValueError("Failed to extract available configurations from the error message.") - + raise ValueError("Failed to get available configurations") else: - raise e # Re-raise other ValueErrors + raise e -# Load function to be used as a connector -def load(dataset_identifier, dataset_split=None): - data = fetch_data_from_huggingface(dataset_identifier.strip(), dataset_split) - if data: - return data - else: + # Processes dataset entries + data = [] + for split in dataset.keys(): + for i, example in enumerate(dataset[split]): + # Creates a unique ULID + ulid = ULID() + example['id'] = str(ulid) + data.append(example) + + + return data + +# Creates AtlasDataset from HF dataset +def hf_atlasdataset(dataset_identifier): + data = get_hfdata(dataset_identifier.strip()) + + + map_name = dataset_identifier.replace('/', '_') + if not data: raise ValueError("No data was found for the provided dataset.") + dataset = AtlasDataset( + map_name, + unique_id_field="id", + ) + + + # Convert all booleans and lists to strings + for entry in data: + for key, value in entry.items(): + if isinstance(value, bool): + entry[key] = str(value) + elif isinstance(value, list): + entry[key] = ' '.join(map(str, value)) + elif isinstance(value, np.ndarray): + entry[key] = ' '.join(map(str, value.flatten())) + elif hasattr(value, 'tolist'): + entry[key] = ' '.join(map(str, value.tolist())) + else: + entry[key] = str(value) + + + dataset.add_data(data=data) + + + return dataset + + + diff --git a/examples/HF_example_usage.py b/examples/HF_example_usage.py index 6078b779..408349f4 100644 --- a/examples/HF_example_usage.py +++ b/examples/HF_example_usage.py @@ -1,10 +1,22 @@ -from nomic.connectors import huggingface_connecter -import logging -# Example source url: https://huggingface.co/datasets/allenai/quartz -#Takes last two parts of url to get allenai/quartz -atlas_dataset = huggingface_connecter.load('allenai/quartz') +from nomic_connector import hf_atlasdataset + + +if __name__ == "__main__": + dataset_identifier = input("Enter Hugging Face dataset identifier: ").strip() + + + try: + atlas_dataset = hf_atlasdataset(dataset_identifier) + print(f"AtlasDataset has been created for '{dataset_identifier}'") + except ValueError as e: + print(f"Error creating AtlasDataset: {e}") + + + + + + + -atlas_dataset.create_index(topic_model=True, embedding_model='NomicEmbed') -logging.info("Atlas dataset has been loaded and indexed successfully.") From d30529dfb6628334ac890cebc1a52d2ce740b664 Mon Sep 17 00:00:00 2001 From: Abhi Somala Date: Tue, 9 Jul 2024 15:49:12 -0400 Subject: [PATCH 09/12] removed print statments in example --- examples/HF_example_usage.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/HF_example_usage.py b/examples/HF_example_usage.py index 408349f4..33cb0de8 100644 --- a/examples/HF_example_usage.py +++ b/examples/HF_example_usage.py @@ -1,6 +1,6 @@ -from nomic_connector import hf_atlasdataset - +from huggingface_connecter import hf_atlasdataset +import logging if __name__ == "__main__": dataset_identifier = input("Enter Hugging Face dataset identifier: ").strip() @@ -8,9 +8,11 @@ try: atlas_dataset = hf_atlasdataset(dataset_identifier) - print(f"AtlasDataset has been created for '{dataset_identifier}'") + logging.info(f"AtlasDataset has been created for '{dataset_identifier}'") except ValueError as e: - print(f"Error creating AtlasDataset: {e}") + logging.error(f"Error creating AtlasDataset: {e}") + + From 829d7df55528ea17a9a7dc1010b665b264b8aa12 Mon Sep 17 00:00:00 2001 From: Abhi Somala Date: Wed, 10 Jul 2024 11:55:07 -0400 Subject: [PATCH 10/12] renamed file, lot of updates including using arrow format batch processing and getting config without parsing through error message --- connectors/huggingface_connecter.py | 77 ----------------------- connectors/huggingface_connector.py | 94 +++++++++++++++++++++++++++++ examples/HF_example_usage.py | 2 +- 3 files changed, 95 insertions(+), 78 deletions(-) delete mode 100644 connectors/huggingface_connecter.py create mode 100644 connectors/huggingface_connector.py diff --git a/connectors/huggingface_connecter.py b/connectors/huggingface_connecter.py deleted file mode 100644 index 03d9afc5..00000000 --- a/connectors/huggingface_connecter.py +++ /dev/null @@ -1,77 +0,0 @@ -from datasets import load_dataset -from nomic import AtlasDataset -from ulid import ULID -import numpy as np - -# Gets data from HF dataset -def get_hfdata(dataset_identifier): - try: - # Loads dataset without specifying config - dataset = load_dataset(dataset_identifier) - except ValueError as e: - # Handles error messages - error_message = str(e) - if "Please pick one among the available configs" in error_message: - try: - available_configs_start = error_message.index("['") + 2 - available_configs_end = error_message.index("']") - available_configs = error_message[available_configs_start:available_configs_end].split("', '") - dataset = load_dataset(dataset_identifier, available_configs[0], trust_remote_code=True) - except ValueError: - raise ValueError("Failed to get available configurations") - else: - raise e - - - # Processes dataset entries - data = [] - for split in dataset.keys(): - for i, example in enumerate(dataset[split]): - # Creates a unique ULID - ulid = ULID() - example['id'] = str(ulid) - data.append(example) - - - return data - -# Creates AtlasDataset from HF dataset -def hf_atlasdataset(dataset_identifier): - data = get_hfdata(dataset_identifier.strip()) - - - map_name = dataset_identifier.replace('/', '_') - if not data: - raise ValueError("No data was found for the provided dataset.") - - - dataset = AtlasDataset( - map_name, - unique_id_field="id", - ) - - - # Convert all booleans and lists to strings - for entry in data: - for key, value in entry.items(): - if isinstance(value, bool): - entry[key] = str(value) - elif isinstance(value, list): - entry[key] = ' '.join(map(str, value)) - elif isinstance(value, np.ndarray): - entry[key] = ' '.join(map(str, value.flatten())) - elif hasattr(value, 'tolist'): - entry[key] = ' '.join(map(str, value.tolist())) - else: - entry[key] = str(value) - - - dataset.add_data(data=data) - - - return dataset - - - - - diff --git a/connectors/huggingface_connector.py b/connectors/huggingface_connector.py new file mode 100644 index 00000000..556b59e8 --- /dev/null +++ b/connectors/huggingface_connector.py @@ -0,0 +1,94 @@ +from datasets import load_dataset, get_dataset_split_names +from nomic import AtlasDataset +import numpy as np +import pandas as pd +import pyarrow as pa + + +# Gets data from HF dataset +def get_hfdata(dataset_identifier): + try: + # Loads dataset without specifying config + dataset = load_dataset(dataset_identifier) + except ValueError as e: + # Grabs available configs and loads dataset using it + configs = get_dataset_split_names(dataset_identifier) + config = configs[0] + dataset = load_dataset(dataset_identifier, config, trust_remote_code=True, streaming=True, split=config + "[:100000]") + + + # Processes dataset entries using Arrow + id_counter = 0 + data = [] + for split in dataset.keys(): + for example in dataset[split]: + # Adds a sequential ID + example['id'] = str(id_counter) + id_counter += 1 + data.append(example) + + + # Convert the data list to an Arrow table + table = pa.Table.from_pandas(pd.DataFrame(data)) + + + return table + + +# Converts booleans, lists etc to strings +def convert_to_string(value): + if isinstance(value, bool): + return str(value) + elif isinstance(value, list): + return ' '.join(map(convert_to_string, value)) + elif isinstance(value, np.ndarray): + return ' '.join(map(str, value.flatten())) + elif hasattr(value, 'tolist'): + return ' '.join(map(str, value.tolist())) + else: + return str(value) + + +# Processes Arrow table and converts necessary fields to strings +def process_table(table): + # Converts columns with complex types to strings + for col in table.schema.names: + column = table[col].to_pandas() + if column.dtype == np.bool_ or column.dtype == object or isinstance(column[0], (list, np.ndarray)): + column = column.apply(convert_to_string) + table = table.set_column(table.schema.get_field_index(col), col, pa.array(column)) + + + return table + + +# Creates AtlasDataset from HF dataset +def hf_atlasdataset(dataset_identifier): + table = get_hfdata(dataset_identifier.strip()) + + + map_name = dataset_identifier.replace('/', '_') + if not table: + raise ValueError("No data was found for the provided dataset.") + + + dataset = AtlasDataset( + map_name, + unique_id_field="id", + ) + + + # Ensures all values are converted to strings + processed_table = process_table(table) + + + # Adds data to the AtlasDataset + dataset.add_data(data=processed_table.to_pandas().to_dict(orient='records')) + + + return dataset + + + + + diff --git a/examples/HF_example_usage.py b/examples/HF_example_usage.py index 33cb0de8..f43d0e8b 100644 --- a/examples/HF_example_usage.py +++ b/examples/HF_example_usage.py @@ -1,5 +1,5 @@ -from huggingface_connecter import hf_atlasdataset +from huggingface_connector import hf_atlasdataset import logging if __name__ == "__main__": From 8bf9c5256bfccf0ff470653c4107e787e15e9a4a Mon Sep 17 00:00:00 2001 From: Abhi Somala Date: Thu, 11 Jul 2024 10:46:03 -0400 Subject: [PATCH 11/12] edits to connector and changed example usage --- connectors/huggingface_connector.py | 82 ++++++++++++++--------------- examples/HF_example_usage.py | 26 ++++----- 2 files changed, 53 insertions(+), 55 deletions(-) diff --git a/connectors/huggingface_connector.py b/connectors/huggingface_connector.py index 556b59e8..7298674b 100644 --- a/connectors/huggingface_connector.py +++ b/connectors/huggingface_connector.py @@ -1,27 +1,36 @@ from datasets import load_dataset, get_dataset_split_names from nomic import AtlasDataset -import numpy as np -import pandas as pd import pyarrow as pa +import pyarrow.compute as pc # Gets data from HF dataset -def get_hfdata(dataset_identifier): +def get_hfdata(dataset_identifier, split="train", limit=100000): try: - # Loads dataset without specifying config - dataset = load_dataset(dataset_identifier) + # Determine available splits + splits = get_dataset_split_names(dataset_identifier) + if split not in splits: + # Use the first available split if the specified split is not found + split = splits[0] + + + # Load the dataset + dataset = load_dataset(dataset_identifier, split=split, streaming=True) except ValueError as e: - # Grabs available configs and loads dataset using it - configs = get_dataset_split_names(dataset_identifier) - config = configs[0] - dataset = load_dataset(dataset_identifier, config, trust_remote_code=True, streaming=True, split=config + "[:100000]") + # Parses through error message if the get_dataset_split_names function doesn't work + if "Please pick one among the available configs" in str(e): + # Gets available config and selects first one + available_configs = str(e).split("['")[1].split("']")[0].split("', '") + dataset = load_dataset(dataset_identifier, available_configs[0], split=f"{split}[:{limit}]", trust_remote_code=True) + else: + raise e # Processes dataset entries using Arrow id_counter = 0 data = [] - for split in dataset.keys(): - for example in dataset[split]: + if dataset: + for example in dataset: # Adds a sequential ID example['id'] = str(id_counter) id_counter += 1 @@ -29,46 +38,33 @@ def get_hfdata(dataset_identifier): # Convert the data list to an Arrow table - table = pa.Table.from_pandas(pd.DataFrame(data)) - - + table = pa.Table.from_pylist(data) return table -# Converts booleans, lists etc to strings -def convert_to_string(value): - if isinstance(value, bool): - return str(value) - elif isinstance(value, list): - return ' '.join(map(convert_to_string, value)) - elif isinstance(value, np.ndarray): - return ' '.join(map(str, value.flatten())) - elif hasattr(value, 'tolist'): - return ' '.join(map(str, value.tolist())) - else: - return str(value) - - -# Processes Arrow table and converts necessary fields to strings +# Function to convert complex types to strings using Arrow def process_table(table): # Converts columns with complex types to strings for col in table.schema.names: - column = table[col].to_pandas() - if column.dtype == np.bool_ or column.dtype == object or isinstance(column[0], (list, np.ndarray)): - column = column.apply(convert_to_string) - table = table.set_column(table.schema.get_field_index(col), col, pa.array(column)) - - + column = table[col] + if pa.types.is_boolean(column.type): + table = table.set_column(table.schema.get_field_index(col), col, pc.cast(column, pa.string())) + elif pa.types.is_list(column.type): + if pa.types.is_struct(column.type.value_type): + new_column = pc.list_flatten(column).cast(pa.string()) + table = table.set_column(table.schema.get_field_index(col), col, new_column) + else: + table = table.set_column(table.schema.get_field_index(col), col, pc.cast(column, pa.string())) + elif pa.types.is_dictionary(column.type): + table = table.set_column(table.schema.get_field_index(col), col, pc.cast(column, pa.string())) return table # Creates AtlasDataset from HF dataset -def hf_atlasdataset(dataset_identifier): - table = get_hfdata(dataset_identifier.strip()) - - +def hf_atlasdataset(dataset_identifier, split="train", limit=100000): + table = get_hfdata(dataset_identifier.strip(), split, limit) map_name = dataset_identifier.replace('/', '_') - if not table: + if table.num_rows == 0: raise ValueError("No data was found for the provided dataset.") @@ -78,12 +74,12 @@ def hf_atlasdataset(dataset_identifier): ) - # Ensures all values are converted to strings + # Process the table to ensure all complex types are converted to strings processed_table = process_table(table) - # Adds data to the AtlasDataset - dataset.add_data(data=processed_table.to_pandas().to_dict(orient='records')) + # Add data to the AtlasDataset + dataset.add_data(data=processed_table) return dataset diff --git a/examples/HF_example_usage.py b/examples/HF_example_usage.py index f43d0e8b..8c0d7b13 100644 --- a/examples/HF_example_usage.py +++ b/examples/HF_example_usage.py @@ -1,22 +1,24 @@ - +import argparse from huggingface_connector import hf_atlasdataset -import logging - -if __name__ == "__main__": - dataset_identifier = input("Enter Hugging Face dataset identifier: ").strip() - - - try: - atlas_dataset = hf_atlasdataset(dataset_identifier) - logging.info(f"AtlasDataset has been created for '{dataset_identifier}'") - except ValueError as e: - logging.error(f"Error creating AtlasDataset: {e}") +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Create an AtlasDataset from a Hugging Face dataset.') + parser.add_argument('--dataset_identifier', type=str, required=True, help='The Hugging Face dataset identifier') + parser.add_argument('--split', type=str, default="train", help='The dataset split to use (default: train)') + parser.add_argument('--limit', type=int, default=100000, help='The maximum number of examples to load (default: 100000)') + args = parser.parse_args() + try: + atlas_dataset = hf_atlasdataset(args.dataset_identifier, args.split, args.limit) + print(f"AtlasDataset has been created for '{args.dataset_identifier}'") + except ValueError as e: + print(f"Error creating AtlasDataset: {e}") + except Exception as e: + print(f"An unexpected error occurred: {e}") From 9ae14f42ed2de9aad1386f93bf75883349eb6b6c Mon Sep 17 00:00:00 2001 From: Abhi Somala Date: Thu, 18 Jul 2024 12:35:31 -0400 Subject: [PATCH 12/12] more updates to HF connector and example --- connectors/huggingface_connector.py | 58 +++++++++-------------------- examples/HF_example_usage.py | 7 +--- 2 files changed, 19 insertions(+), 46 deletions(-) diff --git a/connectors/huggingface_connector.py b/connectors/huggingface_connector.py index 7298674b..3a37a6bf 100644 --- a/connectors/huggingface_connector.py +++ b/connectors/huggingface_connector.py @@ -3,45 +3,27 @@ import pyarrow as pa import pyarrow.compute as pc - # Gets data from HF dataset def get_hfdata(dataset_identifier, split="train", limit=100000): - try: - # Determine available splits - splits = get_dataset_split_names(dataset_identifier) - if split not in splits: - # Use the first available split if the specified split is not found - split = splits[0] - - - # Load the dataset - dataset = load_dataset(dataset_identifier, split=split, streaming=True) - except ValueError as e: - # Parses through error message if the get_dataset_split_names function doesn't work - if "Please pick one among the available configs" in str(e): - # Gets available config and selects first one - available_configs = str(e).split("['")[1].split("']")[0].split("', '") - dataset = load_dataset(dataset_identifier, available_configs[0], split=f"{split}[:{limit}]", trust_remote_code=True) - else: - raise e + splits = get_dataset_split_names(dataset_identifier) + dataset = load_dataset(dataset_identifier, split=split, streaming=True) + if not dataset: + raise ValueError("No dataset was found for the provided identifier and split.") # Processes dataset entries using Arrow id_counter = 0 data = [] - if dataset: - for example in dataset: - # Adds a sequential ID - example['id'] = str(id_counter) - id_counter += 1 - data.append(example) - + for example in dataset: + # Adds a sequential ID + example['id'] = str(id_counter) + id_counter += 1 + data.append(example) # Convert the data list to an Arrow table table = pa.Table.from_pylist(data) return table - # Function to convert complex types to strings using Arrow def process_table(table): # Converts columns with complex types to strings @@ -50,16 +32,19 @@ def process_table(table): if pa.types.is_boolean(column.type): table = table.set_column(table.schema.get_field_index(col), col, pc.cast(column, pa.string())) elif pa.types.is_list(column.type): - if pa.types.is_struct(column.type.value_type): - new_column = pc.list_flatten(column).cast(pa.string()) - table = table.set_column(table.schema.get_field_index(col), col, new_column) - else: - table = table.set_column(table.schema.get_field_index(col), col, pc.cast(column, pa.string())) + new_column = [] + for item in column: + if pa.types.is_struct(column.type.value_type): + # Flatten the struct and cast as string for each row + flattened = ", ".join(str(sub_item.as_py()) for sub_item in item.values) + new_column.append(flattened) + else: + new_column.append(str(item)) + table = table.set_column(table.schema.get_field_index(col), col, pa.array(new_column, pa.string())) elif pa.types.is_dictionary(column.type): table = table.set_column(table.schema.get_field_index(col), col, pc.cast(column, pa.string())) return table - # Creates AtlasDataset from HF dataset def hf_atlasdataset(dataset_identifier, split="train", limit=100000): table = get_hfdata(dataset_identifier.strip(), split, limit) @@ -67,24 +52,17 @@ def hf_atlasdataset(dataset_identifier, split="train", limit=100000): if table.num_rows == 0: raise ValueError("No data was found for the provided dataset.") - dataset = AtlasDataset( map_name, unique_id_field="id", ) - # Process the table to ensure all complex types are converted to strings processed_table = process_table(table) - # Add data to the AtlasDataset dataset.add_data(data=processed_table) - return dataset - - - diff --git a/examples/HF_example_usage.py b/examples/HF_example_usage.py index 8c0d7b13..d7990a96 100644 --- a/examples/HF_example_usage.py +++ b/examples/HF_example_usage.py @@ -1,17 +1,14 @@ import argparse from huggingface_connector import hf_atlasdataset - if __name__ == "__main__": parser = argparse.ArgumentParser(description='Create an AtlasDataset from a Hugging Face dataset.') - parser.add_argument('--dataset_identifier', type=str, required=True, help='The Hugging Face dataset identifier') + parser.add_argument('--dataset_identifier', type=str, required=True, help='The Hugging Face dataset identifier (e.g., "username/dataset_name")') parser.add_argument('--split', type=str, default="train", help='The dataset split to use (default: train)') parser.add_argument('--limit', type=int, default=100000, help='The maximum number of examples to load (default: 100000)') - args = parser.parse_args() - try: atlas_dataset = hf_atlasdataset(args.dataset_identifier, args.split, args.limit) print(f"AtlasDataset has been created for '{args.dataset_identifier}'") @@ -22,5 +19,3 @@ - -