diff --git a/package.json b/package.json index b0370354b..20a307b84 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,9 @@ }, "dependencies": { "depcheck": "^1.4.3", - "typedoc-plugin-missing-exports": "^2.1.0" + "typedoc-plugin-missing-exports": "^2.1.0", + "@web3-storage/w3cli": "^3.0.1", + "ipfs-car": "^0.9.1" }, "pnpm": { "peerDependencyRules": { diff --git a/packages/w3up-python-client/.gitignore b/packages/w3up-python-client/.gitignore new file mode 100644 index 000000000..54080ef5f --- /dev/null +++ b/packages/w3up-python-client/.gitignore @@ -0,0 +1,6 @@ +.env +*.car +/storacha_uploader.egg-info +/myenv +node_modules + diff --git a/packages/w3up-python-client/README.md b/packages/w3up-python-client/README.md new file mode 100644 index 000000000..207251c11 --- /dev/null +++ b/packages/w3up-python-client/README.md @@ -0,0 +1,96 @@ +# Storacha Uploader + +A tool to create CAR files and upload content to Storacha decentralized storage. + +## Install + +```bash +# Requires Python 3.6+, Node.js and npm +pip install . +``` + +## Configure + +Create a `.env` file with your credentials: +``` +X_AUTH_SECRET_HEADER=your_auth_secret +AUTHORIZATION_HEADER=your_authorization +SPACE_DID=your_space_did +``` + +## Usage + +```bash +storacha-uploader myfile.txt +``` + +This converts the file to CAR format, uploads it to Storacha, and provides an access link. + + +## Instructions to be used over the http bridge + +Below instructions can be used over the http bridge by adding them to the list.json file. + +```bash +{ + "tasks": [ + /* + [ + "access/delegate", + "did:key:z6Mkabc123", + { "delegate": "" } + ], + [ + "space/info", + "did:key:z6Mkabc123" + ], + [ + "space/allocate", + "did:key:z6Mkabc123", + { "allocation": "" } + ], + [ + "store/add", + "did:key:z6Mkabc123", + { "root": { "/": "" }, "shards": [] } + ], + [ + "store/get", + "did:key:z6Mkabc123", + { "root": { "/": "" } } + ], + [ + "store/remove", + "did:key:z6Mkabc123", + { "root": { "/": "" } } + ], + [ + "store/list", + "did:key:z6Mkabc123" + ], + */ + [ + "upload/add", + "did:key:z6Mkabc123", + { "root": { "/": "" }, "shards": [] } + ] + /* + , + [ + "upload/list", + "did:key:z6Mkabc123", + {} + ], + [ + "upload/remove", + "did:key:z6Mkabc123", + { "root": { "/": "" } } + ], + [ + "usage/report", + "did:key:z6Mkabc123" + ] + */ + ] +} +``` \ No newline at end of file diff --git a/packages/w3up-python-client/list.json b/packages/w3up-python-client/list.json new file mode 100644 index 000000000..927378de5 --- /dev/null +++ b/packages/w3up-python-client/list.json @@ -0,0 +1,9 @@ +{ + "tasks": [ + [ + "upload/list", + "did:key:z6MksbhMHBFohHmhhDNkLVSRr9FgSJRU4rHrP74ggr2EiRt7", + {} + ] + ] + } diff --git a/packages/w3up-python-client/myfile.txt b/packages/w3up-python-client/myfile.txt new file mode 100644 index 000000000..911f61af0 --- /dev/null +++ b/packages/w3up-python-client/myfile.txt @@ -0,0 +1 @@ +Test conversion to car file \ No newline at end of file diff --git a/packages/w3up-python-client/requirements.txt b/packages/w3up-python-client/requirements.txt new file mode 100644 index 000000000..c46ba2516 --- /dev/null +++ b/packages/w3up-python-client/requirements.txt @@ -0,0 +1,3 @@ +requests +python-dotenv +setuptools \ No newline at end of file diff --git a/packages/w3up-python-client/setup.py b/packages/w3up-python-client/setup.py new file mode 100644 index 000000000..8db997523 --- /dev/null +++ b/packages/w3up-python-client/setup.py @@ -0,0 +1,54 @@ +import subprocess +import sys +import platform +from setuptools import setup, find_packages + +def check_w3cli(): + print("šŸ” Checking for w3cli...") + try: + subprocess.run(["w3", "--version"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print("āœ… w3cli is already installed.") + return True + except FileNotFoundError: + print("āŒ w3cli not found.") + print("šŸ“¦ Installing w3cli globally...") + try: + subprocess.run("npm install -g @web3-storage/w3cli", shell=True, check=True) + print("āœ… w3cli installed successfully.") + return True + except subprocess.CalledProcessError as e: + print(f"āŒ Failed to install w3cli: {e}") + print("Please install it manually with: npm install -g @web3-storage/w3cli") + return False + +w3cli_installed = check_w3cli() + +with open('requirements.txt') as f: + requirements = f.read().splitlines() + +setup( + name="storacha_uploader", + version="0.1.0", + packages=find_packages(), + install_requires=requirements, + entry_points={ + "console_scripts": [ + "storacha-uploader=storacha_uploader:main", + ], + }, + author="Amit Pandey", + author_email="a_pandey1@ce.iitr.ac.in", + description="A tool to automate IPFS DAG creation, CAR file generation, and Web3 Storage upload.", + long_description=open("README.md").read() if sys.version_info[0] >= 3 else "", + long_description_content_type="text/markdown", + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + python_requires=">=3.6", +) + +print("\nāœ… Setup completed.") +print("šŸ“ Note: Make sure Node.js and npm are installed on your system.") +print("šŸš€ You can now use the 'storacha-uploader' command to upload files to Storacha network.") diff --git a/packages/w3up-python-client/storacha_uploader.py b/packages/w3up-python-client/storacha_uploader.py new file mode 100644 index 000000000..2deacc654 --- /dev/null +++ b/packages/w3up-python-client/storacha_uploader.py @@ -0,0 +1,136 @@ +import os +import sys +import json +import subprocess +import requests +import tempfile +import platform +from dotenv import load_dotenv + +def run_command(command): + try: + result = subprocess.check_output(command, shell=True, text=True).strip() + return result + except subprocess.CalledProcessError as e: + print(f"āŒ Error: {e}") + return None + +def send_request_to_the_bridge(json_payload, auth_secret, authorization, endpoint): + headers = { + "X_AUTH_SECRET": auth_secret, + "Authorization": authorization, + "Content-Type": "application/json" + } + + response = requests.post(endpoint, headers=headers, data=json_payload) + + if response.status_code == 200: + response_data = response.json() + upload_url = response_data.get('upload_url') + if not upload_url: + raise ValueError("Upload URL not found in the response.") + return upload_url + else: + raise requests.HTTPError(f"Upload registration failed: {response.status_code} {response.text}") + +def upload_car_file(upload_url, car_file_path): + with open(car_file_path, 'rb') as f: + response = requests.put(upload_url, data=f) + if response.status_code not in [200, 201]: + raise requests.HTTPError(f"CAR file upload failed: {response.status_code} {response.text}") + +def upload_add(cid): + payload = { + "tasks": [ + ["upload/add", "did:key:z6Mkabc123", {"root": {"/": cid}, "shards": []}] + ] + } + return send_request(payload) + + +def upload_list(): + payload = { + "tasks": [ + ["upload/list", "did:key:z6Mkabc123", {}] + ] + } + return send_request(payload) + + +def upload_remove(cid): + payload = { + "tasks": [ + ["upload/remove", "did:key:z6Mkabc123", {"root": {"/": cid}}] + ] + } + return send_request(payload) + + +def send_request(payload,endpoint,headers): + response = requests.post(endpoint, headers=headers, json=payload) + if response.status_code == 200: + return response.json() + else: + raise requests.HTTPError(f"Request failed: {response.status_code} {response.text}") + +def main(): + load_dotenv() + + auth_secret = os.getenv("X_AUTH_SECRET_HEADER") + authorization = os.getenv("AUTHORIZATION_HEADER") + endpoint = os.getenv("HTTPS_ENDPOINT", "https://up.storacha.network/bridge") + + if len(sys.argv) < 2: + print("āŒ Error: Please provide an operation (upload_add, upload_list, upload_remove) and optional CID.") + print("Usage: python storacha_uploader.py ") + sys.exit(1) + + operation = sys.argv[1] + cid = sys.argv[2] if len(sys.argv) > 2 else None + + operations = { + "upload_add": upload_add, + "upload_list": upload_list, + "upload_remove": upload_remove, + } + + if operation in operations: + result = operations[operation](cid if "list" not in operation else None) + print(json.dumps(result, indent=2)) + else: + print(f"Unknown operation: {operation}") + + + file_path = sys.argv[1] + + space_did = os.getenv("SPACE_DID") + if not space_did: + print("āŒ Error: SPACE_DID not found in .env file.") + print("Please add SPACE_DID=your_space_did to your .env file.") + sys.exit(1) + + print(f"šŸ”‘ Using Space DID: {space_did}") + + car_file = f"{file_path}.car" + ipfs_car_cmd = ["ipfs-car", "pack", file_path, "--output", car_file] + + try: + result = subprocess.run(ipfs_car_cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + cid = result.stdout.decode().strip().split('\n')[-1] + print(f"āœ… CAR file created: {car_file} with CID: {cid}") + except subprocess.CalledProcessError as e: + print(f"āŒ Error creating CAR file: {e.stderr.decode().strip()}") + sys.exit(1) + + upload_url = send_request_to_the_bridge(auth_secret, authorization, endpoint) + upload_car_file(upload_url, car_file) + upload_add(cid) + print(f"āœ… File uploaded successfully to Storacha network.") + print(f"āœ… Access your file at: https://{cid}.ipfs.w3s.link") + print(f"āœ… Response details: {json.dumps(result, indent=2)}") + + except Exception as e: + print(f"āŒ Error: {e}") + sys.exit(1) +if __name__ == "__main__": + main()