diff --git a/README.md b/README.md index f614606..d304b09 100644 --- a/README.md +++ b/README.md @@ -1,81 +1,105 @@ -# ansol +# Autoclock RPC -### machine setting -* this process works best on latitude machines - * because the initial state of the machine is cleaner - * disks are named consistently (nvme01, nvme0n2) - * ubuntu installed (preferably ubuntu 20.04, 22.04) - this won't work with centos etc since they don't use aptitude by default - * the login user being ubuntu helps (all the solana operations are done using the solana user that the ansible playbook creates) - * ubuntu should be in the sudoers list - * clean unmounted disks. if your root is on one of partitions and you pass it as an argument, this could potentially be disastrous. +### What is it good for? -* all the above are satisfied by a fresh latitude launch +The goal of the Autoclock RPC ansible playbook is to have you caught up on the Solana blockchain within 15 minutes, assuming you have a capable server and your SSH key ready. It formats/raids/mounts disks, sets up swap, ramdisk (optional), downloads from snapshot and restarts everything. It is currently configured for a Latitude.sh s3.large.x86 (see "Optimal Machine Settings" below), but we hope to adapt it more widely later on. For a more catch-all ansible playbook and in depth guide on RPC's refer to https://github.com/rpcpool/solana-rpc-ansible -* you can launch latitude machines here https://www.latitude.sh/pricing -* recommend the s3.large.x86 - it is one of the most performant nodes for staying at tip +### Optimal Machine Settings -* Specs - * 24 cores or more - * 512 GB ram if you want to use ramdisk/tmpfs and store the accounts db in RAM (we use 300 GB for ram disk). without tmpfs, the ram requirement can be significantly lower. ~256 GB - * 3-4 TB (multiple disks is ok i.e. 2x 1.9TB because the ansible playbook stripes them together) +- Our Latitude.sh s3.large.x86 server starts with the settings below, which we prefer because: -### step 1: ssh into your machine + - the initial state of the machine is cleaner than others that we have tried + - disks are named consistently (nvme01, nvme0n2) + - ubuntu installed (preferably ubuntu 20.04, 22.04) - this won't work with centos, etc. since they don't use aptitude by default + - the login user being ubuntu helps (all the solana operations are done using the solana user that the ansible playbook creates) + - ubuntu is in the sudoer's list + - unmounted disks are clean - if your root is on one of partitions and you pass it as an argument, this could be disastrous +- All the above are satisfied by a fresh s3.large.x86 launch found here: https://www.latitude.sh/pricing +- Zen3 AMD Epyc’s such as the 7443p are considered some of the most performant nodes for keeping up with the tip of the chain at the moment, and support large amounts of RAM. + +- Recommended RPC Specs + - 24 cores or more + - 512 GB RAM if you want to use ramdisk/tmpfs and store the accounts db in RAM (we use 300 GB for ram disk). without tmpfs, the ram requirement can be significantly lower (~256 GB) + - 3-4 TB (multiple disks is okay - i.e. 2x 1.9TB - because the ansible playbook stripes them together) + +### Step 1: SSH into your machine + +### Step 2: Start a screen session -### step 2: start a screen session ``` screen -S sol ``` -### step 3: install ansible +### Step 3: Install ansible + ``` -sudo apt-get install ansible -y +sudo apt-get update && sudo apt-get install ansible -y ``` -### step 4: clone the anssol repository +### Step 4: Clone the autoclock-rpc repository + ``` -git clone https://github.com/dubbelosix/ansol.git +git clone https://github.com/overclock-validator/autoclock-rpc.git ``` -### step 5: cd into the ansol folder +### Step 5: cd into the autoclock-rpc folder + ``` -cd ansol +cd autoclock-rpc ``` -### step 6: run the ansible command -* this command can take between 10-20 minutes based on the specs of the machine -* it takes long because it does everything necessary to start the validator (format disks, checkout the solana repo and build, download the latest snapshot etc) +### Step 6: Run the ansible command + +- this command can take between 10-20 minutes based on the specs of the machine +- it takes long because it does everything necessary to start the validator (format disks, checkout the solana repo and build it, download the latest snapshot, etc.) +- make sure that the solana_version is up to date (see below) +- check the values set in `defaults/main.yml` and update to the values you want + ``` -time ansible-playbook runner.yaml --extra-vars='{"solana_version": "v1.13.4", "swap_mb":100000,"raw_disk_list":["/dev/nvme0n1","/dev/nvme1n1"],"setup_disks":true,"download_snapshot":true,"ramdisk_size":300}' +time ansible-playbook runner.yaml ``` -#### params explained -* solana_version: which version of solana do we want to run -* swap_mb: megabytes of swap. can set this to 50% of RAM or even lower. 100 GB is fine on a 512 GB RAM machine (variable value is in MB so 100000) -* raw_disk_list: the list of currently unmounted disks that will be wiped, raided, formatted with ext4 and then mounted to /mnt -* ramdisk_size: this is optional and only necessary if you want to use ramdisk for the validator - carves out a large portion of the RAM to store the accountsdb. On a 512 GB RAM instance, this can be set to 300 GB (variable value is in GB so 300) +#### ~ Parameters explained ~ + +- solana_version: the version of solana that we want to run. Check the Solana Tech discord’s mb-announcements channel for the recommended version. +- swap_mb: megabytes of swap. This can be set this to 50% of RAM or even lower. 100 GB is fine on a 512 GB RAM machine (variable value is in MB so 100000) +- raw_disk_list: the list of currently unmounted disks that will be wiped, raided, formatted with ext4 and then mounted to /mnt +- ramdisk_size: this is optional and only necessary if you want to use ramdisk for the validator - carves out a large portion of the RAM to store the accountsdb. On a 512 GB RAM instance, this can be set to 300 GB (variable value is in GB so 300) +- solana_installer: whether to install solana from the installer. If set to false it will build solana cli from the solana github + +### Step 7: Once ansible finishes, switch to the solana user with: -### step 7: after ansible finishes -switch to the solana user with ``` sudo su - solana ``` -### step 8: and check the validator status with + +### Step 8: Check the status + ``` -/mnt/solana/target/release/solana-validator --ledger /mnt/solana-ledger monitor +source ~/.profile +solana-validator --ledger /mnt/solana-ledger monitor ledger monitor Ledger location: /mnt/solana-ledger ⠉ Validator startup: SearchingForRpcService... ``` -### Initially the monitor should just show. this will last for a few minutes and is normal +#### Initially the monitor should just show the below message which will last for a few minutes and is normal: + ``` ⠉ Validator startup: SearchingForRpcService... ``` -### after a while, the message at the terminal should change to + +#### After a while, the message at the terminal should change to something similar to this: + ``` ⠐ 00:08:26 | Processed Slot: 156831951 | Confirmed Slot: 156831951 | Finalized Slot: 156831917 | Full Snapshot Slot: 156813730 | ``` -If you see the message above, then everything is working fine! Gratz. you have a new RPC server and you can visit the URL at http://xx.xx.xx.xx:8899/ +#### Check whether the RPC is caught up with the rest of the cluster with: + +``` +solana catchup --our-localhost +``` +If you see the message above, then everything is working fine! Gratz. You have a new RPC server and you can visit the URL at http://xx.xx.xx.xx:8899/ diff --git a/defaults/main.yml b/defaults/main.yml new file mode 100644 index 0000000..d0100bb --- /dev/null +++ b/defaults/main.yml @@ -0,0 +1,9 @@ +--- +# defaults file for Solana RPC +solana_version: "v1.13.5" +raw_disk_list: ["/dev/nvme1n1", "/dev/nvme2n1"] +setup_disks: "true" +download_snapshot: "true" +ramdisk_size: 300 +swap_mb: "100000" + diff --git a/files/download_start.sh b/files/download_start.sh new file mode 100644 index 0000000..fc33020 --- /dev/null +++ b/files/download_start.sh @@ -0,0 +1,4 @@ +#!/bin/bash +sudo systemctl stop sol.service +python3 /mnt/snapshot-finder.py --snapshot_path /mnt/solana-snapshots +sudo systemctl start sol.service \ No newline at end of file diff --git a/montip.py b/files/montip.py similarity index 93% rename from montip.py rename to files/montip.py index 00b0227..a2433c9 100644 --- a/montip.py +++ b/files/montip.py @@ -3,7 +3,7 @@ import json import time -GENESYSGO = "https://ssc-dao.genesysgo.net/" +MAINNET = "https://api.mainnet-beta.solana.com" LOCAL = "http://localhost:8899" PAYLOAD = {"jsonrpc":"2.0","id":1, "method":"getSlot", "params":[{"commitment":"processed"}]} @@ -22,7 +22,7 @@ def get_slot(req, jsondata, result, idx): tlist = [] reqlist = [] resultlist = [0,0] - for c,u in enumerate([LOCAL, GENESYSGO]): + for c,u in enumerate([LOCAL, MAINNET]): req = urllib.request.Request(u) req.add_header('Content-Type', 'application/json; charset=utf-8') req.add_header('Content-Length', content_len) diff --git a/files/restart.sh b/files/restart.sh new file mode 100644 index 0000000..4be5120 --- /dev/null +++ b/files/restart.sh @@ -0,0 +1,7 @@ +#!/bin/bash +if [ $# -eq 0 ] + then + python3 /mnt/snapcheck.py +fi +sudo systemctl stop sol.service +sudo systemctl start sol.service diff --git a/snapcheck.py b/files/snapcheck.py similarity index 100% rename from snapcheck.py rename to files/snapcheck.py diff --git a/snapshot-finder.py b/files/snapshot-finder.py similarity index 97% rename from snapshot-finder.py rename to files/snapshot-finder.py index d1fade4..6c6e7c3 100644 --- a/snapshot-finder.py +++ b/files/snapshot-finder.py @@ -28,7 +28,7 @@ parser.add_argument('--max_download_speed', type=int, help='Maximum snapshot download speed in megabytes - https://github.com/c29r3/solana-snapshot-finder/issues/11. Example: --max_download_speed 192') parser.add_argument('--max_latency', default=40, type=int, help='The maximum value of latency (milliseconds). If latency > max_latency --> skip') -parser.add_argument('--version', type=str, help='version of the snapshot required') +# parser.add_argument('--version', type=str, help='version of the snapshot required') parser.add_argument('--with_private_rpc', action="store_true", help='Enable adding and checking RPCs with the --private-rpc option.This slow down checking and searching but potentially increases' ' the number of RPCs from which snapshots can be downloaded.') parser.add_argument('--measurement_time', default=7, type=int, help='Time in seconds during which the script will measure the download speed') @@ -52,7 +52,7 @@ MAX_DOWNLOAD_SPEED_MB = args.max_download_speed SPEED_MEASURE_TIME_SEC = args.measurement_time MAX_LATENCY = args.max_latency -VERSION = args.version +# VERSION = args.version SNAPSHOT_PATH = args.snapshot_path if args.snapshot_path[-1] != '/' else args.snapshot_path[:-1] NUM_OF_MAX_ATTEMPTS = args.num_of_retries SLEEP_BEFORE_RETRY = args.sleep @@ -403,17 +403,16 @@ def main_worker(): if rpc_node["snapshot_address"] in unsuitable_servers: logger.info(f'Rpc node already in unsuitable list --> skip {rpc_node["snapshot_address"]}') continue - - logger.info("checking version") - try: - if not version_check(rpc_node["snapshot_address"], VERSION): - logger.info("version check failed") - continue - except: - import traceback - print(traceback.format_exc()) - print("broken") - logger.info("version check succeeded") + # logger.info("checking version") + # try: + # if not version_check(rpc_node["snapshot_address"], VERSION): + # logger.info("version check failed") + # continue + # except: + # import traceback + # print(traceback.format_exc()) + # print("broken") + # logger.info("version check succeeded") down_speed_bytes = measure_speed(url=rpc_node["snapshot_address"], measure_time=SPEED_MEASURE_TIME_SEC) down_speed_mb = convert_size(down_speed_bytes) if down_speed_bytes < MIN_DOWNLOAD_SPEED_MB * 1e6: diff --git a/sol.service b/files/sol.service similarity index 50% rename from sol.service rename to files/sol.service index b583333..a8c3e95 100644 --- a/sol.service +++ b/files/sol.service @@ -10,9 +10,8 @@ RestartSec=1 User=solana LimitNOFILE=1000000 LogRateLimitIntervalSec=0 -Environment="PATH=/mnt/solana/target/release/:/mnt/solana/target/release/:/home/solana/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/snap/bin:/home/solana/.local/bin/:/home/solana/.local/bin/:/home/solana/.local/bin/" +Environment="PATH=/mnt/solana/target/release/:/home/solana/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/snap/bin:/home/solana/.local/bin/:/home/solana/.local/bin/:/home/solana/.local/bin/" ExecStart=/home/solana/validator.sh [Install] WantedBy=multi-user.target - diff --git a/files/validator.sh b/files/validator.sh new file mode 100644 index 0000000..f541508 --- /dev/null +++ b/files/validator.sh @@ -0,0 +1,16 @@ +#!/bin/bash +export SOLANA_METRICS_CONFIG="host=https://metrics.solana.com:8086,db=mainnet-beta,u=mainnet-beta_write,p=password" +exec /mnt/solana/target/release/solana-validator --identity /home/solana/rpc_node.json \ +--full-rpc-api \ +--rpc-port 8899 \ +--entrypoint entrypoint.mainnet-beta.solana.com:8001 \ +--limit-ledger-size \ +--log /mnt/logs/solana-validator.log \ +--accounts /mnt/solana-accounts \ +--ledger /mnt/solana-ledger \ +--snapshots /mnt/solana-snapshots \ +--no-snapshot-fetch \ +--known-validator 7Np41oeYqPefeNQEHSv1UDhYrehxin3NStELsSKCT4K2 \ +--known-validator GdnSyH3YtwcxFvQrVVJMm1JhTS4QVX7MFsX56uJLUfiZ \ +--known-validator DE1bawNcRJB9rVm3buyMVfr8mBEoyyu73NBovf2oXJsJ \ +--known-validator CakcnaRDHka2gXyfbEd2d3xsvkJkqsLw2akB3zsN1D2S \ No newline at end of file diff --git a/fstab.yaml b/fstab.yaml deleted file mode 100644 index 3af44b2..0000000 --- a/fstab.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- name: Add the below lines - blockinfile: - path: /etc/fstab - state: present - block: | - /mnt/swapfile swap swap defaults 0 0 - /dev/md0 /mnt ext4 defaults,nofail,discard 0 0 diff --git a/restart.sh b/restart.sh deleted file mode 100644 index 5e139e3..0000000 --- a/restart.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -if [ $# -eq 0 ] - then - python3 /mnt/snapcheck.py -fi -sudo systemctl stop sol.service -sudo pkill -9 -f solana-sys-tuner -sudo $(command -v /mnt/solana/target/release/solana-sys-tuner) --user $(whoami) > sys-tuner.log 2>&1 & -sudo systemctl start sol.service diff --git a/runner.yaml b/runner.yaml index 2fcfd78..75dd414 100644 --- a/runner.yaml +++ b/runner.yaml @@ -1,58 +1,6 @@ --- - - name: "playbook runner" - hosts: localhost - connection: local - tasks: - - - name: var check - ansible.builtin.import_tasks: var_check.yaml - - - name: setup disks - ansible.builtin.import_tasks: disks.yaml - when: setup_disks|default(false)|bool == true - - - name: create user - ansible.builtin.import_tasks: user.yaml - - - name: install dependencies - ansible.builtin.import_tasks: deps.yaml - - - name: folders - ansible.builtin.import_tasks: dirs.yaml - - - name: swap - ansible.builtin.import_tasks: swap.yaml - - - name: ramdisk - ansible.builtin.import_tasks: ramdisk.yaml - when: ramdisk_size is defined - - - name: logrotate - ansible.builtin.import_tasks: rotate.yaml - - - name: git - ansible.builtin.import_tasks: git.yaml - - - name: solana keygen - ansible.builtin.import_tasks: keygen.yaml - - - name: file setup - ansible.builtin.import_tasks: file_setup.yaml - - - name: snapshot download - ansible.builtin.import_tasks: snapshot_downloader.yaml - when: download_snapshot|default(true)|bool == true - - - name: restart without waiting - become: true - become_user: root - shell: /home/solana/restart.sh 1 - when: download_snapshot|default(true)|bool == true - - - name: restart with waiting - become: true - become_user: root - shell: /home/solana/restart.sh - when: download_snapshot|default(true)|bool == false - - +- name: "playbook runner" + hosts: localhost + connection: local + roles: + - role: "./" diff --git a/deps.yaml b/tasks/deps.yaml similarity index 97% rename from deps.yaml rename to tasks/deps.yaml index 0802707..0368ec6 100644 --- a/deps.yaml +++ b/tasks/deps.yaml @@ -49,4 +49,4 @@ when: cargo_exists is failed shell: /tmp/sh.rustup.rs -y args: - executable: /bin/bash + executable: /bin/bash \ No newline at end of file diff --git a/dirs.yaml b/tasks/dirs.yaml similarity index 97% rename from dirs.yaml rename to tasks/dirs.yaml index 1bf7689..fe0f159 100644 --- a/dirs.yaml +++ b/tasks/dirs.yaml @@ -4,7 +4,7 @@ state: directory owner: solana group: solana - mode: '0777' + mode: "0777" become: true become_user: root diff --git a/disks.yaml b/tasks/disks.yaml similarity index 73% rename from disks.yaml rename to tasks/disks.yaml index d065a14..b5de48b 100644 --- a/disks.yaml +++ b/tasks/disks.yaml @@ -4,12 +4,12 @@ apt: update_cache: yes pkg: - - mdadm + - mdadm - name: check raid already exists become: true become_user: root - shell: fdisk -l | grep -e 'md0' -e 'md127' + shell: fdisk -l | grep -e 'md' ignore_errors: yes register: raid_exists @@ -52,4 +52,17 @@ become_user: root shell: mount /dev/{{ raid_name.stdout }} /mnt when: mount_mnt.rc != 0 - + +- name: extract raid params + shell: sudo blkid | grep md | grep -oh \".*\" | sed -e 's/"//g' | awk '{print "UUID="$1" /mnt ext4 defaults 1 1"}' + register: raid_config + +- name: add raid to fstab + become: true + become_user: root + lineinfile: + dest: /etc/fstab + state: present + line: "{{ raid_config.stdout_lines[0] }}" + insertbefore: 'mnt' + firstmatch: yes diff --git a/file_setup.yaml b/tasks/file_setup.yaml similarity index 60% rename from file_setup.yaml rename to tasks/file_setup.yaml index 0a05794..0fb5b90 100644 --- a/file_setup.yaml +++ b/tasks/file_setup.yaml @@ -1,37 +1,47 @@ - name: sol validator setup become: true become_user: solana - template: - src: validator.j2 + copy: + src: validator.sh dest: /home/solana/validator.sh owner: solana group: solana - mode: '0755' + mode: "0755" - name: setup sol.service become: true become_user: root - template: + copy: src: sol.service dest: /etc/systemd/system/sol.service owner: root group: root - mode: '0755' + mode: "0755" - name: copy restart.sh become: true become_user: solana - template: - src: restart.sh + copy: + src: restart.sh dest: /home/solana/restart.sh owner: solana group: solana - mode: '0755' + mode: "0755" + +- name: copy restart.sh + become: true + become_user: solana + copy: + src: download_restart.sh + dest: /home/solana/download_restart.sh + owner: solana + group: solana + mode: "0755" - name: copy snapcheck.py become: true become_user: solana - template: + copy: src: snapcheck.py dest: /mnt/snapcheck.py owner: solana @@ -40,8 +50,18 @@ - name: copy snapshot-finder.py become: true become_user: solana - template: + copy: src: snapshot-finder.py dest: /mnt/snapshot-finder.py owner: solana group: solana + +- name: copy montip.py + become: true + become_user: solana + copy: + src: montip.py + dest: /mnt/montip.py + owner: solana + group: solana + diff --git a/git.yaml b/tasks/git.yaml similarity index 72% rename from git.yaml rename to tasks/git.yaml index 3c921e3..a15c480 100644 --- a/git.yaml +++ b/tasks/git.yaml @@ -9,7 +9,7 @@ - name: build solana become: true become_user: solana - shell: source /home/solana/.cargo/env && cargo build --release + shell: source /home/solana/.cargo/env && /mnt/solana/cargo build --release args: chdir: /mnt/solana - executable: /bin/bash + executable: /bin/bash \ No newline at end of file diff --git a/keygen.yaml b/tasks/keygen.yaml similarity index 81% rename from keygen.yaml rename to tasks/keygen.yaml index d13ebf1..28cf1e5 100644 --- a/keygen.yaml +++ b/tasks/keygen.yaml @@ -3,4 +3,4 @@ become_user: solana shell: /mnt/solana/target/release/solana-keygen new --no-bip39-passphrase -o /home/solana/rpc_node.json args: - creates: /home/solana/rpc_node.json + creates: /home/solana/rpc_node.json \ No newline at end of file diff --git a/tasks/main.yaml b/tasks/main.yaml new file mode 100644 index 0000000..76ad0bb --- /dev/null +++ b/tasks/main.yaml @@ -0,0 +1,55 @@ +--- +# tasks file for Solana RPC +- name: var check + include_tasks: var_check.yaml + +- name: setup disks + include_tasks: disks.yaml + when: setup_disks|default(false)|bool == true + +- name: create user + include_tasks: user.yaml + +- name: install dependencies + include_tasks: deps.yaml + +- name: folders + include_tasks: dirs.yaml + +- name: swap + include_tasks: swap.yaml + +- name: ramdisk + include_tasks: ramdisk.yaml + when: ramdisk_size is defined + +- name: logrotate + include_tasks: rotate.yaml + +- name: git + include_tasks: git.yaml + +- name: solana keygen + include_tasks: keygen.yaml + +- name: file setup + include_tasks: file_setup.yaml + +- name: tune host + include_tasks: tuner.yaml + +- name: snapshot download + include_tasks: snapshot_downloader.yaml + when: download_snapshot|default(true)|bool == true + +- name: restart without waiting + become: true + become_user: root + shell: /home/solana/restart.sh 1 + when: download_snapshot|default(true)|bool == true + +- name: restart with waiting + become: true + become_user: root + shell: /home/solana/restart.sh + when: download_snapshot|default(true)|bool == false diff --git a/ramdisk.yaml b/tasks/ramdisk.yaml similarity index 73% rename from ramdisk.yaml rename to tasks/ramdisk.yaml index 3cf7ec8..5365bfb 100644 --- a/ramdisk.yaml +++ b/tasks/ramdisk.yaml @@ -4,11 +4,11 @@ shell: mount -t tmpfs -o size={{ ramdisk_size }}G tmpfs /mnt/solana-accounts when: ansible_memtotal_mb >= 1500*ramdisk_size -- name: add ramdisk to fstab +- name: add ramdisk to fstab become: true become_user: root lineinfile: dest: /etc/fstab state: present - line: 'tmpfs /mnt/solana-accounts tmpfs rw,size={{ ramdisk_size }}G,user=solana 0 0' + line: "tmpfs /mnt/solana-accounts tmpfs rw,size={{ ramdisk_size }}G,user=solana 0 0" when: ansible_memtotal_mb >= 1500*ramdisk_size diff --git a/rotate.yaml b/tasks/rotate.yaml similarity index 96% rename from rotate.yaml rename to tasks/rotate.yaml index fd15643..97ffbeb 100644 --- a/rotate.yaml +++ b/tasks/rotate.yaml @@ -4,7 +4,7 @@ file: path: /etc/logrotate.d/sol state: touch - mode: '0644' + mode: "0644" owner: root group: root modification_time: preserve diff --git a/snapshot_downloader.yaml b/tasks/snapshot_downloader.yaml similarity index 84% rename from snapshot_downloader.yaml rename to tasks/snapshot_downloader.yaml index b43199e..8fc787b 100644 --- a/snapshot_downloader.yaml +++ b/tasks/snapshot_downloader.yaml @@ -3,12 +3,11 @@ become_user: solana ansible.builtin.pip: name: - - tqdm + - tqdm - requests executable: pip3 - name: download latest snapshot become: true become_user: solana - shell: python3 /mnt/snapshot-finder.py --snapshot_path /mnt/solana-snapshots --max_latency 1000 - + shell: python3 /mnt/snapshot-finder.py --snapshot_path /mnt/solana-snapshots diff --git a/swap.yaml b/tasks/swap.yaml similarity index 70% rename from swap.yaml rename to tasks/swap.yaml index b260994..a8799cf 100644 --- a/swap.yaml +++ b/tasks/swap.yaml @@ -1,16 +1,18 @@ - name: Create swap file become: true become_user: root - command: dd if=/dev/zero of=/mnt/swapfile bs=1024 count={{ swap_mb }}k - creates="/mnt/swapfile" + command: + dd if=/dev/zero of=/mnt/swapfile bs=1024 count={{ swap_mb }}k + creates="/mnt/swapfile" - name: Change swap file permissions become: true become_user: root - file: path="/mnt/swapfile" - owner=root - group=root - mode=0600 + file: + path=/mnt/swapfile + owner=root + group=root + mode=0600 - name: Check swap file type become: true @@ -28,12 +30,12 @@ become: true become_user: root mount: name=none - src=/mnt/swapfile - fstype=swap - opts=sw - passno=0 - dump=0 - state=present + src=/mnt/swapfile + fstype=swap + opts=sw + passno=0 + dump=0 + state=present - name: swap check become: true diff --git a/tasks/tuner.yaml b/tasks/tuner.yaml new file mode 100644 index 0000000..b82ad9f --- /dev/null +++ b/tasks/tuner.yaml @@ -0,0 +1,71 @@ +- name: Set sysctl performance variables + become: true + become_user: root + shell: + # Note: this overwrite the file, if you want to append replace EOM with EOF + cmd: | + bash -c 'cat >> /etc/sysctl.conf <<- EOM + # set minimum, default, and maximum tcp buffer sizes (10k, 87.38k (linux default), 12M resp) + net.ipv4.tcp_rmem=10240 87380 12582912 + net.ipv4.tcp_wmem=10240 87380 12582912 + # Enable TCP westwood for kernels greater than or equal to 2.6.13 + net.ipv4.tcp_congestion_control=westwood + net.ipv4.tcp_fastopen=3 + net.ipv4.tcp_timestamps=0 + net.ipv4.tcp_sack=1 + net.ipv4.tcp_low_latency=1 + # Enable fast recycling TIME-WAIT sockets + # net.ipv4.tcp_tw_recycle = 1 this is in solana tuner, but fails, changing to below, see - https://djangocas.dev/blog/troubleshooting-tcp_tw_recycle-no-such-file-or-directory/ + net.ipv4.tcp_tw_reuse = 1 + # dont cache ssthresh from previous connection + net.ipv4.tcp_no_metrics_save = 1 + net.ipv4.tcp_moderate_rcvbuf = 1 + + # kernel Tunes + kernel.timer_migration=0 + kernel.hung_task_timeout_secs=30 + # A suggested value for pid_max is 1024 * <# of cpu cores/threads in system> + kernel.pid_max=49152 + + # vm.tuning + vm.swappiness=30 + vm.max_map_count=2000000 + vm.stat_interval=10 + vm.dirty_ratio=40 + vm.dirty_background_ratio=10 + vm.min_free_kbytes = 3000000 + vm.dirty_expire_centisecs=36000 + vm.dirty_writeback_centisecs=3000 + vm.dirtytime_expire_seconds=43200 + + # solana systuner + net.core.rmem_max=134217728 + net.core.rmem_default=134217728 + net.core.wmem_max=134217728 + net.core.wmem_default=134217728 + EOM' + args: + executable: /bin/bash + +- name: Reload sysctl + become: true + become_user: root + shell: sysctl -p + +- name: Set performance governor + become: true + become_user: root + shell: | + echo 'GOVERNOR="performance"' | tee /etc/default/cpufrequtils + echo "performance" | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor + args: + executable: /bin/bash + +- name: Set performance governor bare metal + become: true + become_user: root + shell: | + echo "performance" | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor + args: + executable: /bin/bash + ignore_errors: True diff --git a/user.yaml b/tasks/user.yaml similarity index 71% rename from user.yaml rename to tasks/user.yaml index 7ad27d8..bb1a511 100644 --- a/user.yaml +++ b/tasks/user.yaml @@ -23,3 +23,10 @@ regexp: '^solana' line: 'solana ALL=(ALL) NOPASSWD: ALL' validate: 'visudo -cf %s' + +- name: Ensure solana build is in path for solana user + become: true + become_user: root + lineinfile: + path: /home/solana/.bashrc + line: 'export PATH=/mnt/solana/target/release:$PATH' diff --git a/var_check.yaml b/tasks/var_check.yaml similarity index 57% rename from var_check.yaml rename to tasks/var_check.yaml index caa59a8..37a0384 100644 --- a/var_check.yaml +++ b/tasks/var_check.yaml @@ -12,3 +12,8 @@ - "{{ raw_disk_list }}" when: setup_disks is defined and (setup_disks | bool) +- name: validate some variables + assert: + that: + - ((ramdisk_size is defined) and (ansible_memtotal_mb >= 1500*ramdisk_size) or ramdisk_size is not defined) + msg: machine ram is too low for ram diskq. comment out ramdisk_size in defaults/main.yml \ No newline at end of file diff --git a/test_runner.yaml b/test_runner.yaml index 3923bfd..987066b 100644 --- a/test_runner.yaml +++ b/test_runner.yaml @@ -1,9 +1,7 @@ --- - - name: "playbook runner" - hosts: localhost - connection: local - tasks: - - - name: var checks - ansible.builtin.import_tasks: var_check.yaml - +- name: "playbook runner" + hosts: localhost + connection: local + tasks: + - name: var checks + ansible.builtin.import_tasks: var_check.yaml diff --git a/validator.j2 b/validator.j2 deleted file mode 100644 index e1c1c0d..0000000 --- a/validator.j2 +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -/mnt/solana/target/release/solana-validator --identity /home/solana/rpc_node.json --rpc-port 8899 --full-rpc-api --entrypoint entrypoint.mainnet-beta.solana.com:8001 --limit-ledger-size --log /mnt/logs/solana-validator.log --accounts /mnt/solana-accounts --ledger /mnt/solana-ledger --snapshots /mnt/solana-snapshots --no-snapshot-fetch