From b730ff055ca78a1c039059c984e8dfbbf9d9befa Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Tue, 9 Sep 2025 15:20:30 -0400 Subject: [PATCH 01/12] Add GitHub Actions workflows, update documentation, and restructure project files - Introduced new workflows for build, documentation, and publishing to PyPI. - Added API reference documentation. - Updated README with installation and usage instructions. - Replaced `requirements_dev.txt` and `setup.cfg` with `pyproject.toml` for dependency management. - Removed obsolete files and updated package structure. --- .claude/settings.local.json | 9 ++ .github/workflows/build.yml | 117 +++++++++++++++++++ .github/workflows/docs.yml | 60 ++++++++++ .github/workflows/publish.yml | 208 ++++++++++++++++++++++++++++++++++ MANIFEST.in | 14 --- README.md | 198 +++++++++++++++++++++++--------- docs/api.rst | 20 ++++ docs/conf.py | 20 +++- docs/index.rst | 2 +- docs/installation.rst | 10 +- docs/usage.rst | 71 +++++++++++- pyproject.toml | 113 ++++++++++++++++++ requirements_dev.txt | 12 -- setup.cfg | 18 --- setup.py | 53 --------- tests/test_trademan.py | 2 +- tox.ini | 19 ---- trademan.code-workspace | 7 -- trademan/__init__.py | 5 + 19 files changed, 769 insertions(+), 189 deletions(-) create mode 100644 .claude/settings.local.json create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/publish.yml delete mode 100644 MANIFEST.in create mode 100644 docs/api.rst create mode 100644 pyproject.toml delete mode 100644 requirements_dev.txt delete mode 100644 setup.cfg delete mode 100644 setup.py delete mode 100644 tox.ini delete mode 100644 trademan.code-workspace diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..9e2c6c8 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,9 @@ +{ + "permissions": { + "allow": [ + "Bash(python:*)" + ], + "deny": [], + "ask": [] + } +} \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..260505b --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,117 @@ +name: Build and Test + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main ] + +jobs: + check-version: + runs-on: ubuntu-latest + outputs: + version-exists: ${{ steps.check-release.outputs.exists }} + version: ${{ steps.get-version.outputs.version }} + files-changed: ${{ steps.check-changes.outputs.changed }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get version from pyproject.toml + id: get-version + run: | + VERSION=$(python -c "import tomli; print(tomli.load(open('pyproject.toml', 'rb'))['project']['version'])") + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Current version: $VERSION" + + - name: Check if release exists + id: check-release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if gh release view "v${{ steps.get-version.outputs.version }}" >/dev/null 2>&1; then + echo "exists=true" >> $GITHUB_OUTPUT + echo "Release v${{ steps.get-version.outputs.version }} already exists" + else + echo "exists=false" >> $GITHUB_OUTPUT + echo "Release v${{ steps.get-version.outputs.version }} does not exist" + fi + + - name: Check if Python files changed + id: check-changes + if: steps.check-release.outputs.exists == 'true' + run: | + # Get the tag for this version + TAG="v${{ steps.get-version.outputs.version }}" + + # Check if any Python files or pyproject.toml changed since the tag + if git diff --name-only $TAG HEAD | grep -E '\.(py|toml)$'; then + echo "changed=true" >> $GITHUB_OUTPUT + echo "Python files have changed since release $TAG" + exit 1 + else + echo "changed=false" >> $GITHUB_OUTPUT + echo "No Python files changed since release $TAG" + fi + + build: + needs: check-version + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build tomli + + - name: Install package with dev dependencies + run: | + pip install -e .[dev] + + - name: Lint with flake8 + run: | + flake8 trademan tests --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 trademan tests --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Format check with black + run: | + black --check --diff trademan tests + + - name: Test with pytest + run: | + pytest tests/ -v --cov=trademan --cov-report=xml + + - name: Upload coverage to Codecov + if: matrix.python-version == '3.11' + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests + name: codecov-umbrella + + - name: Build package + run: | + python -m build + + - name: Check package + run: | + pip install twine + twine check dist/* + + - name: Upload build artifacts + if: matrix.python-version == '3.11' + uses: actions/upload-artifact@v3 + with: + name: dist + path: dist/ \ No newline at end of file diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..886e90d --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,60 @@ +name: Documentation + +on: + push: + branches: [ main ] + paths: + - 'docs/**' + - 'trademan/**' + - 'README.md' + - 'pyproject.toml' + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[dev] + + - name: Build documentation + run: | + cd docs + make html + + - name: Setup Pages + uses: actions/configure-pages@v3 + + - name: Upload artifact + uses: actions/upload-pages-artifact@v2 + with: + path: 'docs/_build/html' + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v2 \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..36e9f5d --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,208 @@ +name: Publish to PyPI + +on: + push: + tags: + - "v*" + workflow_dispatch: + inputs: + create_release: + description: 'Create GitHub release' + required: true + default: 'true' + type: boolean + +jobs: + check-version: + runs-on: ubuntu-latest + outputs: + version-exists: ${{ steps.check-release.outputs.exists }} + version: ${{ steps.get-version.outputs.version }} + files-changed: ${{ steps.check-changes.outputs.changed }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get version from pyproject.toml + id: get-version + run: | + VERSION=$(python -c "import tomli; print(tomli.load(open('pyproject.toml', 'rb'))['project']['version'])") + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Current version: $VERSION" + + - name: Check if release exists + id: check-release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if gh release view "v${{ steps.get-version.outputs.version }}" >/dev/null 2>&1; then + echo "exists=true" >> $GITHUB_OUTPUT + echo "Release v${{ steps.get-version.outputs.version }} already exists" + else + echo "exists=false" >> $GITHUB_OUTPUT + echo "Release v${{ steps.get-version.outputs.version }} does not exist" + fi + + - name: Check if Python files changed + id: check-changes + if: steps.check-release.outputs.exists == 'true' + run: | + TAG="v${{ steps.get-version.outputs.version }}" + if git diff --name-only $TAG HEAD | grep -E '\.(py|toml)$'; then + echo "changed=true" >> $GITHUB_OUTPUT + echo "Python files have changed since release $TAG" + exit 1 + else + echo "changed=false" >> $GITHUB_OUTPUT + echo "No Python files changed since release $TAG" + fi + + test: + needs: check-version + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.11"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tomli + pip install -e .[dev] + + - name: Test with pytest + run: | + pytest tests/ -v + + build: + needs: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build twine tomli + + - name: Build package + run: | + python -m build + + - name: Check package + run: | + twine check dist/* + + - name: Upload build artifacts + uses: actions/upload-artifact@v3 + with: + name: dist + path: dist/ + + publish: + needs: [check-version, build] + runs-on: ubuntu-latest + environment: release + permissions: + id-token: write + contents: write + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download build artifacts + uses: actions/download-artifact@v3 + with: + name: dist + path: dist/ + + - name: Create GitHub Release + if: needs.check-version.outputs.version-exists == 'false' || github.event.inputs.create_release == 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + VERSION="v${{ needs.check-version.outputs.version }}" + + # Generate release notes from git log + if git tag -l "$VERSION" | grep -q "$VERSION"; then + LAST_TAG=$(git tag --sort=-version:refname | grep -v "$VERSION" | head -n 1) + else + LAST_TAG=$(git tag --sort=-version:refname | head -n 1) + fi + + echo "# Release $VERSION" > RELEASE_NOTES.md + echo "" >> RELEASE_NOTES.md + + if [ -n "$LAST_TAG" ]; then + echo "## Changes since $LAST_TAG" >> RELEASE_NOTES.md + git log --pretty=format:"- %s (%h)" "$LAST_TAG"..HEAD >> RELEASE_NOTES.md + else + echo "## Initial Release" >> RELEASE_NOTES.md + echo "- First release of trademan" >> RELEASE_NOTES.md + fi + + gh release create "$VERSION" \ + --title "Release $VERSION" \ + --notes-file RELEASE_NOTES.md \ + --draft=false \ + --prerelease=false \ + dist/* + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + print-hash: true + + docs: + needs: publish + runs-on: ubuntu-latest + permissions: + contents: read + pages: write + id-token: write + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[dev] + + - name: Build documentation + run: | + cd docs + make html + + - name: Setup Pages + uses: actions/configure-pages@v3 + + - name: Upload artifact + uses: actions/upload-pages-artifact@v2 + with: + path: 'docs/_build/html' + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v2 \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 4b613f2..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,14 +0,0 @@ -include AUTHORS.rst -include CONTRIBUTING.rst -include HISTORY.rst -include LICENSE -include README.rst - -recursive-include tests * -recursive-include *.csv -recursive-include *.png - -recursive-exclude * __pycache__ -recursive-exclude * *.py[co] - -recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif diff --git a/README.md b/README.md index 8bf977b..a1ebbd8 100644 --- a/README.md +++ b/README.md @@ -1,62 +1,150 @@ -### What Is This? -`trademan` is a utility to gather market data and generate optimal portfolios via a CLI interface. Data is provided by [yfinance](https://pypi.org/project/yfinance/) and portfolio optimization is done with [PyPortfolioOpt](https://pyportfolioopt.readthedocs.io/en/latest/). Stock data is cached to a diskcache for a week to prevent excess YFinance calls. - -### How To Install It: -`pip install git+https://github.com/SoundsSerious/trademan.git@v0.1.0` - -### How Do I Use It: -The current interface is provided by cli and matplotlib plots - -##### Download Market Data: -`market_dl` will download snp500 and etfs and save data to a diskcache location customizable via env var: `TRADEMAN_DATA_DIR` - -##### Trademan Portfolio Optimizer: -`trademan` CLI has the following CLI arguments - -```bash -usage: Portfolio Generator [-h] [-risk {covariance,ledoit_wolf}] [-retrn {mean}] - [-opt {sharpe,min_volatility,eff_return,eff_risk}] [-cls {etfs,stocks,all}] [-alloc ALLOC] - [-name NAME] [-filename FILENAME] [-rfr RFR] [-gamma GAMMA] [-cycl-err CYCL_ERR] - [-std-err STD_ERR] [-min-wght MIN_WGHT] [-max-wght MAX_WGHT] [-in INCLUDE] [-ex EXCLUDE] - -optional arguments: - -h, --help show this help message and exit - -risk {covariance,ledoit_wolf} - select the risk model, standard covariance or the extremity filtering `ledoit wolf` model - -retrn {mean} return model - mean historical performance averages - -opt {sharpe,min_volatility,eff_return,eff_risk} - optimization model: maximum risk to return model via sharpe, min_volatility only considers - risk, and efficient models will try to achieve 90 percent of the best performing asset - -cls {etfs,stocks,all} - choose which type of items to trade - -alloc ALLOC choose the amount of money to allocate, this will label the output chart with the number of - shares to purchase - -name NAME add a name to the portfolio, if none provided a randomly generated name will be created - -filename FILENAME where to store the file, by default it will be stored in a dir set by `TRADEMAN_MEDIA_DIR` - -rfr RFR the risk free rate, adjusted per daily returns - -gamma GAMMA the weight regularizer, large values penalize small weight values, make 0 to not penalize - small weights - -cycl-err CYCL_ERR default: 0| penalize new assets returns by a factor of economic cycle: `cycle-err x standard - error x (10/Nyears)^2` - -std-err STD_ERR default: 0| penalize returns by subtracting the `std-err x std-dev` - -min-wght MIN_WGHT assets less than this percent are filtered from the final portfolio - -max-wght MAX_WGHT assets are limited to this max percentage - -in INCLUDE, --include INCLUDE - csv of a strict include on the ticker name - -ex EXCLUDE, --exclude EXCLUDE - csv of a strict exclude on the ticker name -``` - -##### Examples -1. Make A Portfolio Of Best Performing SNP500 Stocks Penalizing Short Lived Assets By 10X With A Max Asset Allocation of 20%, And Determine Number of assets shares to buy with an allocation of $10000. -```bash -trademan -cls stocks -gamma 1 -alloc 10000 -cycl-err 10 -max-wght 0.2 +# Trademan + +[![PyPI version](https://badge.fury.io/py/trademan.svg)](https://badge.fury.io/py/trademan) +[![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) +[![Documentation](https://img.shields.io/badge/docs-latest-brightgreen.svg)](https://ottermatics.github.io/trademan/) + +A Python library and CLI tool for gathering market data and generating optimal portfolios using modern portfolio theory. + +## Features + +- 📊 **Market Data Collection**: Automated downloading and caching of S&P 500 and ETF data via yfinance +- 🎯 **Portfolio Optimization**: Multiple optimization strategies (Sharpe ratio, minimum volatility, etc.) +- 📈 **Visualization**: Generate beautiful portfolio allocation charts +- 🚀 **CLI Interface**: Easy-to-use command-line tools for quick analysis +- 🐍 **Python API**: Full programmatic access for custom workflows +- 💾 **Smart Caching**: Intelligent data caching to minimize API calls + +## Installation + +Install from PyPI: +```bash +pip install trademan +``` + +Install from source: +```bash +pip install git+https://github.com/ottermatics/trademan.git +``` + +## Quick Start + +### CLI Usage + +1. **Download Market Data**: +```bash +market_dl # Downloads S&P 500 and ETF data +``` + +2. **Generate a Portfolio**: +```bash +# Create a Sharpe-optimized portfolio with $10,000 allocation +trademan -cls stocks -alloc 10000 -opt sharpe + +# Minimum volatility ETF portfolio +trademan -cls etfs -opt min_volatility -alloc 100000 -in QQQ,SPY,VTI +``` + +### Python API + +```python +import trademan + +# Get stock data for specific tickers +data = trademan.get_tickers(['AAPL', 'MSFT', 'GOOGL']) + +# Create optimized portfolio +weights = trademan.make_portfolio( + data, + opt='sharpe', # Optimization method + risk='ledoit_wolf', # Risk model + allocate_amount=10000 # Dollar amount +) + +# Visualize the portfolio +fig, ax = trademan.plot_portfolio(weights) +``` + +## CLI Options + +| Option | Description | Default | +|--------|-------------|---------| +| `-cls` | Asset class: `etfs`, `stocks`, `all` | `all` | +| `-opt` | Optimization: `sharpe`, `min_volatility`, `eff_return`, `eff_risk` | `sharpe` | +| `-risk` | Risk model: `covariance`, `ledoit_wolf` | `ledoit_wolf` | +| `-alloc` | Amount to allocate (shows share counts) | None | +| `-gamma` | Weight regularization (higher = more diversified) | 2 | +| `-in` | Include specific tickers (comma-separated) | None | +| `-ex` | Exclude specific tickers (comma-separated) | None | +| `-min-wght` | Minimum weight threshold for assets | 0.01 | +| `-max-wght` | Maximum weight limit per asset | None | + +## Configuration + +Set environment variables to customize data storage: + +```bash +export TRADEMAN_DATA_DIR="/path/to/data" # Market data cache +export TRADEMAN_MEDIA_DIR="/path/to/charts" # Generated charts +``` + +## Examples + +### 1. Best Performing S&P 500 Stocks +Create a portfolio favoring established companies with cycle penalties: + +```bash +trademan -cls stocks -gamma 1 -alloc 10000 -cycl-err 10 -max-wght 0.2 ``` + ![Stocks Portfolio](./media/Stocks.png) +### 2. Low Volatility ETF Portfolio +Generate a conservative ETF allocation: -2. Make A Portfolio Of Least Volatile ETFS: `QQQ,SCHG,VGT,SLV,VIG,SPY,VOO,VUG,IAU,PAVE` and determine the number of stock purchases for a $100000 allocation. ```bash -trademan -cls etfs -gamma 0.1 -alloc 100000 -in QQQ,SCHG,VGT,SLV,VIG,SPY,VOO,VUG,IAU,PAVE -opt min_volatility +trademan -cls etfs -gamma 0.1 -alloc 100000 \ + -in QQQ,SCHG,VGT,SLV,VIG,SPY,VOO,VUG,IAU,PAVE \ + -opt min_volatility ``` + ![ETF Portfolio](./media/ETFS_Min_Volatility.png) + +## How It Works + +1. **Data Collection**: Downloads historical price data using yfinance +2. **Risk Modeling**: Calculates covariance matrices with Ledoit-Wolf shrinkage +3. **Return Estimation**: Uses mean historical returns with optional adjustments +4. **Optimization**: Applies modern portfolio theory via PyPortfolioOpt +5. **Allocation**: Converts weights to discrete share quantities +6. **Visualization**: Creates publication-ready portfolio charts + +## Dependencies + +- **PyPortfolioOpt**: Portfolio optimization algorithms +- **yfinance**: Market data source +- **pandas/numpy**: Data manipulation +- **matplotlib**: Visualization +- **diskcache**: Data caching +- **scikit-learn**: Additional analytics + +## Development + +Install development dependencies: +```bash +pip install -e .[dev] +``` + +Run tests: +```bash +pytest +``` + +## License + +MIT License - see [LICENSE](LICENSE) for details. + +## Contributing + +Contributions welcome! Please read [CONTRIBUTING.rst](CONTRIBUTING.rst) for guidelines. \ No newline at end of file diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 0000000..6bc52b8 --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,20 @@ +API Reference +============= + +This section documents the main functions and classes in the trademan package. + +Data Module +----------- + +.. automodule:: trademan.data + :members: + :undoc-members: + :show-inheritance: + +Portfolio Module +---------------- + +.. automodule:: trademan.portfolio + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index b967a27..ac1de59 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,7 +31,13 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', + 'sphinx.ext.napoleon', + 'sphinx.ext.intersphinx', + 'sphinx_rtd_theme' +] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -47,7 +53,7 @@ # General information about the project. project = 'trademan' -copyright = "2023, Kevin Russell" +copyright = "2024, Kevin Russell" author = "Kevin Russell" # The version info for the project you're documenting, acts as replacement @@ -77,13 +83,21 @@ # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False +# Intersphinx mappings +intersphinx_mapping = { + 'python': ('https://docs.python.org/3/', None), + 'numpy': ('https://numpy.org/doc/stable/', None), + 'pandas': ('https://pandas.pydata.org/docs/', None), + 'matplotlib': ('https://matplotlib.org/stable/', None), +} + # -- Options for HTML output ------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a # theme further. For a list of options available for each theme, see the diff --git a/docs/index.rst b/docs/index.rst index 6e30bc9..d8131b5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,7 +8,7 @@ Welcome to trademan's documentation! readme installation usage - modules + api contributing authors history diff --git a/docs/installation.rst b/docs/installation.rst index 18c386c..ada9f41 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -32,20 +32,20 @@ You can either clone the public repository: .. code-block:: console - $ git clone git://github.com/soundsserious/trademan + $ git clone git://github.com/ottermatics/trademan Or download the `tarball`_: .. code-block:: console - $ curl -OJL https://github.com/soundsserious/trademan/tarball/master + $ curl -OJL https://github.com/ottermatics/trademan/tarball/master Once you have a copy of the source, you can install it with: .. code-block:: console - $ python setup.py install + $ pip install . -.. _Github repo: https://github.com/soundsserious/trademan -.. _tarball: https://github.com/soundsserious/trademan/tarball/master +.. _Github repo: https://github.com/ottermatics/trademan +.. _tarball: https://github.com/ottermatics/trademan/tarball/master diff --git a/docs/usage.rst b/docs/usage.rst index 764662e..6d63787 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -2,6 +2,75 @@ Usage ===== -To use trademan in a project:: +Command Line Interface +====================== + +Trademan provides two main CLI commands: + +Data Collection +--------------- + +Download market data for S&P 500 stocks and ETFs:: + + market_dl + +This downloads and caches stock data using yfinance, storing it in a location customizable via the ``TRADEMAN_DATA_DIR`` environment variable. + +Portfolio Optimization +---------------------- + +Generate optimized portfolios:: + + trademan [OPTIONS] + +Key options include: + +* ``-cls {etfs,stocks,all}``: Choose asset class (default: all) +* ``-opt {sharpe,min_volatility,eff_return,eff_risk}``: Optimization method (default: sharpe) +* ``-risk {covariance,ledoit_wolf}``: Risk model (default: ledoit_wolf) +* ``-alloc AMOUNT``: Money to allocate (shows share quantities) +* ``-gamma GAMMA``: Weight regularization parameter +* ``-in TICKERS``: Include specific tickers (comma-separated) +* ``-ex TICKERS``: Exclude specific tickers (comma-separated) + +Examples:: + + # Best performing S&P 500 stocks with $10,000 allocation + trademan -cls stocks -gamma 1 -alloc 10000 -cycl-err 10 -max-wght 0.2 + + # Minimum volatility ETF portfolio + trademan -cls etfs -opt min_volatility -alloc 100000 -in QQQ,SPY,VTI + +Python API +========== + +Basic usage in Python:: import trademan + + # Get stock data + data = trademan.get_tickers(['AAPL', 'MSFT', 'GOOGL']) + + # Create optimized portfolio + weights = trademan.make_portfolio(data, opt='sharpe', allocate_amount=10000) + + # Plot portfolio + fig, ax = trademan.plot_portfolio(weights) + +Configuration +============= + +Environment Variables +-------------------- + +* ``TRADEMAN_DATA_DIR``: Directory for cached market data (default: temp/trademan/data) +* ``TRADEMAN_MEDIA_DIR``: Directory for generated charts (default: temp/trademan/media) + +Data Caching +------------ + +Market data is cached using diskcache: + +* Performance data: 5 days +* Company info: 30 days +* Failed tickers are tracked to avoid repeated API calls diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..46d113d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,113 @@ +[build-system] +requires = ["setuptools>=45", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "trademan" +version = "0.1.0" +description = "Trade and analyze stocks with portfolio optimization" +readme = "README.md" +requires-python = ">=3.8" +license = {text = "MIT"} +license-files = ["LICENSE"] +authors = [ + {name = "Kevin Russell", email = "kevin@ottermatics.com"} +] +keywords = ["trading", "portfolio", "optimization", "stocks", "finance"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: Financial and Insurance Industry", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Office/Business :: Financial :: Investment", + "Topic :: Scientific/Engineering :: Mathematics", +] +dependencies = [ + "PyPortfolioOpt", + "randomname", + "numpy", + "scipy", + "cvxpy", + "yfinance", + "matplotlib", + "diskcache", + "scikit-learn", + "pandas", + "pytz", +] + +[project.optional-dependencies] +dev = [ + "pytest>=6.0", + "pytest-cov", + "black>=21.7b0", + "flake8>=3.7.8", + "bump2version>=1.0.1", + "wheel", + "twine", + "sphinx>=4.0.0", + "sphinx-rtd-theme", + "coverage>=4.5.4", + "tox>=3.14.0", + "watchdog>=0.9.0", + "tomli>=1.2.0;python_version<'3.11'", +] + +[project.urls] +Homepage = "https://github.com/ottermatics/trademan" +Repository = "https://github.com/ottermatics/trademan" +Issues = "https://github.com/ottermatics/trademan/issues" + +[project.scripts] +market_dl = "trademan.data:main" +trademan = "trademan.portfolio:cli" + +[tool.setuptools.packages.find] +include = ["trademan*"] + +[tool.setuptools.package-data] +trademan = ["*.csv", "*.png"] + +[tool.black] +line-length = 100 +target-version = ['py38'] +include = '\.pyi?$' +exclude = ''' +/( + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | dist +)/ +''' + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = "test_*.py" +python_classes = "Test*" +python_functions = "test_*" +addopts = "-v --tb=short" + +[tool.coverage.run] +source = ["trademan"] +omit = ["*/tests/*"] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise AssertionError", + "raise NotImplementedError", +] \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index 0b055bc..0000000 --- a/requirements_dev.txt +++ /dev/null @@ -1,12 +0,0 @@ -pip==19.2.3 -bump2version==0.5.11 -wheel==0.33.6 -watchdog==0.9.0 -flake8==3.7.8 -tox==3.14.0 -coverage==4.5.4 -Sphinx==1.8.5 -twine==1.14.0 - - -black==21.7b0 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index c2ba63b..0000000 --- a/setup.cfg +++ /dev/null @@ -1,18 +0,0 @@ -[bumpversion] -current_version = 0.1.0 -commit = True -tag = True - -[bumpversion:file:setup.py] -search = version='{current_version}' -replace = version='{new_version}' - -[bumpversion:file:trademan/__init__.py] -search = __version__ = '{current_version}' -replace = __version__ = '{new_version}' - -[bdist_wheel] -universal = 1 - -[flake8] -exclude = docs diff --git a/setup.py b/setup.py deleted file mode 100644 index 8a5e836..0000000 --- a/setup.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python - -"""The setup script.""" - -from setuptools import setup, find_packages - -with open('README.md') as readme_file: - readme = readme_file.read() - -with open('HISTORY.rst') as history_file: - history = history_file.read() - -requirements = ['PyPortfolioOpt','randomname','numpy','scipy','cvxpy','yfinance','matplotlib','diskcache','scikit-learn'] - -test_requirements = [ ] - -setup( - author="Kevin Russell", - author_email='kevin@ottermatics.com', - python_requires='>=3.6', - classifiers=[ - 'Development Status :: 2 - Pre-Alpha', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Natural Language :: English', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - ], - description="trade and analyize stocks", - entry_points={ - 'console_scripts': [ - #'trademan=trademan.cli:main', - 'market_dl=trademan.data:main', - 'trademan=trademan.portfolio:cli' - ], - }, - install_requires=requirements, - license="MIT license", - long_description=readme + '\n\n' + history, - include_package_data=True, - keywords='trademan', - name='trademan', - packages=find_packages(include=['trademan', 'trademan.*']), - package_data={'trademan':['*.csv'], - 'trademan.media':['*.png']}, - test_suite='tests', - tests_require=test_requirements, - url='https://github.com/soundsserious/trademan', - version='0.1.0', - zip_safe=False, -) diff --git a/tests/test_trademan.py b/tests/test_trademan.py index 0e45996..abc42fb 100644 --- a/tests/test_trademan.py +++ b/tests/test_trademan.py @@ -7,7 +7,7 @@ from trademan import data from trademan.portfolio import * -from trademan.trademan.data import get_tickers +import os import subprocess as subp diff --git a/tox.ini b/tox.ini deleted file mode 100644 index a197260..0000000 --- a/tox.ini +++ /dev/null @@ -1,19 +0,0 @@ -[tox] -envlist = py36, py37, py38, flake8 - -[travis] -python = - 3.8: py38 - 3.7: py37 - 3.6: py36 - -[testenv:flake8] -basepython = python -deps = flake8 -commands = flake8 trademan tests - -[testenv] -setenv = - PYTHONPATH = {toxinidir} - -commands = python setup.py test diff --git a/trademan.code-workspace b/trademan.code-workspace deleted file mode 100644 index 2a0ed79..0000000 --- a/trademan.code-workspace +++ /dev/null @@ -1,7 +0,0 @@ -{ - "folders": [ - { - "path": ".." - } - ] -} \ No newline at end of file diff --git a/trademan/__init__.py b/trademan/__init__.py index 9a21a81..bea3e5f 100644 --- a/trademan/__init__.py +++ b/trademan/__init__.py @@ -3,3 +3,8 @@ __author__ = """Kevin Russell""" __email__ = 'kevin@ottermatics.com' __version__ = '0.1.0' + +from .data import get_tickers, get_ticker_perf +from .portfolio import make_portfolio, plot_portfolio + +__all__ = ['get_tickers', 'get_ticker_perf', 'make_portfolio', 'plot_portfolio'] From a865c68398d16ee396df9653bfbc108a95caf2c5 Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Tue, 9 Sep 2025 15:53:37 -0400 Subject: [PATCH 02/12] Set up Python 3.11 and install dependencies in build and publish workflows --- .github/workflows/build.yml | 10 ++++++++++ .github/workflows/publish.yml | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 260505b..9f037e1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,6 +18,16 @@ jobs: with: fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install tomli + # GitHub CLI is pre-installed on GitHub runners + - name: Get version from pyproject.toml id: get-version run: | diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 36e9f5d..8a3bc54 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -24,6 +24,16 @@ jobs: with: fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install tomli + # GitHub CLI is pre-installed on GitHub runners + - name: Get version from pyproject.toml id: get-version run: | From 4889dd387a2632b4c33e6e848f3a3d0154508905 Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Tue, 9 Sep 2025 16:27:17 -0400 Subject: [PATCH 03/12] Update GitHub Actions workflows to use latest action versions --- .github/workflows/build.yml | 2 +- .github/workflows/docs.yml | 4 ++-- .github/workflows/publish.yml | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9f037e1..5646648 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -121,7 +121,7 @@ jobs: - name: Upload build artifacts if: matrix.python-version == '3.11' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: dist path: dist/ \ No newline at end of file diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 886e90d..63acfb4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -44,7 +44,7 @@ jobs: uses: actions/configure-pages@v3 - name: Upload artifact - uses: actions/upload-pages-artifact@v2 + uses: actions/upload-pages-artifact@v3 with: path: 'docs/_build/html' @@ -57,4 +57,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v2 \ No newline at end of file + uses: actions/deploy-pages@v4 \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8a3bc54..18f41d2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -118,7 +118,7 @@ jobs: twine check dist/* - name: Upload build artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: dist path: dist/ @@ -137,7 +137,7 @@ jobs: fetch-depth: 0 - name: Download build artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: dist path: dist/ @@ -209,10 +209,10 @@ jobs: uses: actions/configure-pages@v3 - name: Upload artifact - uses: actions/upload-pages-artifact@v2 + uses: actions/upload-pages-artifact@v3 with: path: 'docs/_build/html' - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v2 \ No newline at end of file + uses: actions/deploy-pages@v4 \ No newline at end of file From 501b62bfd4ed340b4c2325ca54692a91e5bf1f8c Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Tue, 9 Sep 2025 16:44:46 -0400 Subject: [PATCH 04/12] Refactor license declaration in pyproject.toml --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 46d113d..a4ac99f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,8 +8,7 @@ version = "0.1.0" description = "Trade and analyze stocks with portfolio optimization" readme = "README.md" requires-python = ">=3.8" -license = {text = "MIT"} -license-files = ["LICENSE"] +license = "MIT" authors = [ {name = "Kevin Russell", email = "kevin@ottermatics.com"} ] From 71d0c2d75370100048a7d195805e46aa1fd78731 Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Tue, 9 Sep 2025 16:51:40 -0400 Subject: [PATCH 05/12] Update setuptools version and modify license declaration format in pyproject.toml --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a4ac99f..121e6d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=45", "wheel"] +requires = ["setuptools>=61.0.0", "wheel"] build-backend = "setuptools.build_meta" [project] @@ -8,7 +8,7 @@ version = "0.1.0" description = "Trade and analyze stocks with portfolio optimization" readme = "README.md" requires-python = ">=3.8" -license = "MIT" +license = {text = "MIT"} authors = [ {name = "Kevin Russell", email = "kevin@ottermatics.com"} ] From 24dcb89b938a462081ee703e2d5efbd8fc3b414b Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Tue, 9 Sep 2025 17:01:11 -0400 Subject: [PATCH 06/12] Add format-check job to GitHub Actions workflow using Black for code formatting --- .github/workflows/build.yml | 55 +++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5646648..f333835 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,6 +65,51 @@ jobs: echo "No Python files changed since release $TAG" fi + format-check: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Black + run: | + python -m pip install --upgrade pip + pip install black + + - name: Check formatting with Black + id: black_check + continue-on-error: true + run: | + black --check trademan tests/ + + - name: Format code with Black and commit (if needed) + if: steps.black_check.outcome == 'failure' && github.event_name == 'pull_request' + run: | + black trademan tests/ + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git add -A + if git diff --staged --quiet; then + echo "No changes to commit after formatting" + else + git commit -m "Auto-format code with Black" + # Pull any changes that might have been pushed by other jobs + git pull origin ${{ github.head_ref }} --rebase + git push origin HEAD:${{ github.head_ref }} + fi + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + build: needs: check-version runs-on: ubuntu-latest @@ -89,15 +134,7 @@ jobs: run: | pip install -e .[dev] - - name: Lint with flake8 - run: | - flake8 trademan tests --count --select=E9,F63,F7,F82 --show-source --statistics - flake8 trademan tests --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Format check with black - run: | - black --check --diff trademan tests - + - name: Test with pytest run: | pytest tests/ -v --cov=trademan --cov-report=xml From befea5d307ebc3a913f19b422973ca4ec4dcf0de Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Tue, 9 Sep 2025 21:01:29 +0000 Subject: [PATCH 07/12] Auto-format code with Black --- tests/test_trademan.py | 17 ++- trademan/__init__.py | 6 +- trademan/data.py | 131 +++++++++-------- trademan/portfolio.py | 317 +++++++++++++++++++++++++++-------------- 4 files changed, 294 insertions(+), 177 deletions(-) diff --git a/tests/test_trademan.py b/tests/test_trademan.py index abc42fb..c425efa 100644 --- a/tests/test_trademan.py +++ b/tests/test_trademan.py @@ -11,14 +11,19 @@ import subprocess as subp + class TestTrademan(unittest.TestCase): """Tests for `trademan` package.""" def test_market_dl(self): """gets appl""" - dl = data.get_tickers('AAPL') - perf = data.data_db[f'perf/AAPL'] - - def test_cli(self) : - os.system("""trademan -cls etfs -gamma 1 -alloc 100000 -in QQQ,SCHG,VGT,SLV,VIG,SPY,VOO,VUG,IAU,PAVE,NANC""") - os.system("""trademan -cls etfs -gamma 20 -alloc 100000 -in QQQ,SCHG,VGT,SLV,VIG,SPY,VOO,VUG,IAU,PAVE,NANC,KRUZ -cycl-err 100""") \ No newline at end of file + dl = data.get_tickers("AAPL") + perf = data.data_db[f"perf/AAPL"] + + def test_cli(self): + os.system( + """trademan -cls etfs -gamma 1 -alloc 100000 -in QQQ,SCHG,VGT,SLV,VIG,SPY,VOO,VUG,IAU,PAVE,NANC""" + ) + os.system( + """trademan -cls etfs -gamma 20 -alloc 100000 -in QQQ,SCHG,VGT,SLV,VIG,SPY,VOO,VUG,IAU,PAVE,NANC,KRUZ -cycl-err 100""" + ) diff --git a/trademan/__init__.py b/trademan/__init__.py index bea3e5f..ca5555f 100644 --- a/trademan/__init__.py +++ b/trademan/__init__.py @@ -1,10 +1,10 @@ """Top-level package for trademan.""" __author__ = """Kevin Russell""" -__email__ = 'kevin@ottermatics.com' -__version__ = '0.1.0' +__email__ = "kevin@ottermatics.com" +__version__ = "0.1.0" from .data import get_tickers, get_ticker_perf from .portfolio import make_portfolio, plot_portfolio -__all__ = ['get_tickers', 'get_ticker_perf', 'make_portfolio', 'plot_portfolio'] +__all__ = ["get_tickers", "get_ticker_perf", "make_portfolio", "plot_portfolio"] diff --git a/trademan/data.py b/trademan/data.py index f15cf54..565fb81 100644 --- a/trademan/data.py +++ b/trademan/data.py @@ -1,130 +1,135 @@ from shutil import ExecError import diskcache -import os,pathlib +import os, pathlib import pandas as pd import yfinance as yf import numpy as np -import datetime,pytz +import datetime, pytz import logging -import time,random +import time, random import tempfile -FORMAT = '%(asctime)s %(message)s' -logging.basicConfig(level=20,format=FORMAT) -log = logging.getLogger('trade-data') +FORMAT = "%(asctime)s %(message)s" +logging.basicConfig(level=20, format=FORMAT) +log = logging.getLogger("trade-data") this_dir = pathlib.Path(__file__).parent -temp_dir = os.path.join(tempfile.gettempdir(),'trademan') +temp_dir = os.path.join(tempfile.gettempdir(), "trademan") -data_dir_dflt = os.path.join(temp_dir,'data') -data_dir = os.environ.get('TRADEMAN_DATA_DIR',data_dir_dflt) +data_dir_dflt = os.path.join(temp_dir, "data") +data_dir = os.environ.get("TRADEMAN_DATA_DIR", data_dir_dflt) -db_path = os.path.join(data_dir,'assets.db') +db_path = os.path.join(data_dir, "assets.db") data_db = diskcache.Cache(db_path) -media_dir_dflt = os.path.join(temp_dir,'media') -media_dir = os.environ.get('TRADEMAN_MEDIA_DIR',media_dir_dflt) +media_dir_dflt = os.path.join(temp_dir, "media") +media_dir = os.environ.get("TRADEMAN_MEDIA_DIR", media_dir_dflt) -print(f'TRADEMAN DATA: {data_dir}') -print(f'TRADEMAN MEDIA: {media_dir}') -print(f'use `TRADEMAN_MEDIA_DIR` or `TRADEMAN_DATA_DIR` envvars to customize paths') -os.makedirs(data_dir,exist_ok=True) -os.makedirs(media_dir,exist_ok=True) +print(f"TRADEMAN DATA: {data_dir}") +print(f"TRADEMAN MEDIA: {media_dir}") +print(f"use `TRADEMAN_MEDIA_DIR` or `TRADEMAN_DATA_DIR` envvars to customize paths") +os.makedirs(data_dir, exist_ok=True) +os.makedirs(media_dir, exist_ok=True) -ticker_record = 'tickers' +ticker_record = "tickers" -dl_keys = {'perf':lambda x: x.history(period='max',interval='1d'), - 'info':lambda x: x.info} +dl_keys = {"perf": lambda x: x.history(period="max", interval="1d"), "info": lambda x: x.info} -db_set_keys = {'perf':{'expire':3600*24*5}, - 'info':{'expire':3600*24*30}} +db_set_keys = {"perf": {"expire": 3600 * 24 * 5}, "info": {"expire": 3600 * 24 * 30}} + +all_stocks = pd.read_csv(os.path.join(this_dir, "stock_info.csv")) +snp500 = pd.read_csv(os.path.join(this_dir, "snp500_members.csv")) +etfs = pd.read_csv(os.path.join(this_dir, "etfs.csv")) -all_stocks = pd.read_csv(os.path.join(this_dir,'stock_info.csv')) -snp500 = pd.read_csv(os.path.join(this_dir,'snp500_members.csv')) -etfs = pd.read_csv(os.path.join(this_dir,'etfs.csv')) def db_existing_symbols(): - return [s.replace('tickers/','') for s in data_db.iterkeys() if s.startswith('tickers/')] + return [s.replace("tickers/", "") for s in data_db.iterkeys() if s.startswith("tickers/")] + -def _dl(ticker,select=None,delay=5): +def _dl(ticker, select=None, delay=5): """smartly accesses diskcache data if it exists, else download it""" out = {} - failures = data_db.get(f'ticker_failures',[]) + failures = data_db.get(f"ticker_failures", []) if ticker in failures: - log.info(f'skipping previously failed {ticker}') + log.info(f"skipping previously failed {ticker}") return - + try: yfobj = yf.Ticker(ticker) - for dk,func in dl_keys.items(): - - key = f'{dk}/{ticker.lower()}' + for dk, func in dl_keys.items(): + + key = f"{dk}/{ticker.lower()}" if key in data_db: - log.debug(f'db get: {ticker}') + log.debug(f"db get: {ticker}") dlip = data_db[key] else: - dlwait = delay*(1+random.random()/2) - log.info(f'downloading {dk}/{ticker} after: {dlwait}s') + dlwait = delay * (1 + random.random() / 2) + log.info(f"downloading {dk}/{ticker} after: {dlwait}s") time.sleep(dlwait) dlip = func(yfobj) - set_kw = {'tag':dk,**db_set_keys.get(dk,{})} - data_db.set(key,dlip,**set_kw) + set_kw = {"tag": dk, **db_set_keys.get(dk, {})} + data_db.set(key, dlip, **set_kw) - #add to record: + # add to record: get_time = datetime.datetime.now(tz=pytz.utc).isoformat() - data_db.set(f'{ticker_record}/{ticker}', get_time,tag='record') + data_db.set(f"{ticker_record}/{ticker}", get_time, tag="record") if select is None or dk in select: out[dk] = dlip - return out - + return out + except Exception as e: - log.error(e,msg=f'issue getting: {ticker}') + log.error(e, msg=f"issue getting: {ticker}") failures.append(ticker) - data_db[f'ticker_failures'] = failures + data_db[f"ticker_failures"] = failures + def clear_failures(): - data_db[f'ticker_failures'] = [] + data_db[f"ticker_failures"] = [] + def get_ticker_perf(ticker): dat = _dl(ticker) if dat is None: return - dfms = pd.DataFrame(dat['perf']) - - dfms['perf'] = dc = (dfms['Close'] - dfms['Open'])/(dfms['Open']) - dfms['grwth'] = np.cumprod(1+dc)-1 - dfms['highQlow'] = ec = (dfms['High'] - dfms['Low'])/(dfms['High'] + dfms['Low']) + dfms = pd.DataFrame(dat["perf"]) + + dfms["perf"] = dc = (dfms["Close"] - dfms["Open"]) / (dfms["Open"]) + dfms["grwth"] = np.cumprod(1 + dc) - 1 + dfms["highQlow"] = ec = (dfms["High"] - dfms["Low"]) / (dfms["High"] + dfms["Low"]) return dfms + def get_tickers(tickers): - - if isinstance(tickers,str): - tickers =tickers.split(',') - log.info(f'getting {len(tickers)}| {str(list(tickers))[:100]}') + if isinstance(tickers, str): + tickers = tickers.split(",") - data ={} + log.info(f"getting {len(tickers)}| {str(list(tickers))[:100]}") + + data = {} for tick in tickers: dfms = get_ticker_perf(tick) if dfms is not None: data[tick] = dfms - table = pd.concat(data.values(),keys=data.keys()) + table = pd.concat(data.values(), keys=data.keys()) + + return table - return table def main(clear=False): - if clear: + if clear: clear_failures() - #TODO: clear db ect via options - get_tickers(etfs['Symbol']) - get_tickers(snp500['Symbol']) + # TODO: clear db ect via options + get_tickers(etfs["Symbol"]) + get_tickers(snp500["Symbol"]) + -if __name__ == '__main__': - main() \ No newline at end of file +if __name__ == "__main__": + main() diff --git a/trademan/portfolio.py b/trademan/portfolio.py index 30f08ef..3348ca9 100644 --- a/trademan/portfolio.py +++ b/trademan/portfolio.py @@ -8,17 +8,18 @@ import cvxpy as cp import argparse -def get_items(canidates,filtr:list=None): - + +def get_items(canidates, filtr: list = None): + tickers = canidates if filtr: - tickers = list(filter(filtr,canidates)) - print(f'filtered: {set(canidates).difference(set(tickers))}') + tickers = list(filter(filtr, canidates)) + print(f"filtered: {set(canidates).difference(set(tickers))}") df = data.get_tickers(tickers) return df - -def plot_portfolio(weights,shares=None): + +def plot_portfolio(weights, shares=None): """ Plot the portfolio weights as a horizontal bar chart @@ -29,7 +30,7 @@ def plot_portfolio(weights,shares=None): :return: matplotlib axis :rtype: matplotlib.axes """ - fig,ax = subplots(figsize=(6,10)) + fig, ax = subplots(figsize=(6, 10)) desc = sorted(weights.items(), key=lambda x: x[1], reverse=True) labels = [i[0] for i in desc] @@ -39,194 +40,300 @@ def plot_portfolio(weights,shares=None): hbars = ax.barh(y_pos, vals) if shares: - ax.bar_label(hbars,labels=[f'{v}' for v in shares.values()]) + ax.bar_label(hbars, labels=[f"{v}" for v in shares.values()]) ax.set_xlabel("Weight") ax.set_yticks(y_pos) ax.set_yticklabels(labels) ax.invert_yaxis() - return fig,ax + return fig, ax + -def simple_allocate(T,df,w_goal): - dfc = df['Close'].unstack(0) +def simple_allocate(T, df, w_goal): + dfc = df["Close"].unstack(0) prices = dfc[list(w_goal)].iloc[-1] - alloc = pfopt.discrete_allocation.DiscreteAllocation(w_goal, prices,T) + alloc = pfopt.discrete_allocation.DiscreteAllocation(w_goal, prices, T) return alloc.greedy_portfolio(True)[0] -def max_allocate(T,df,w_goal): - dfc = df['Close'].unstack(0) - prices = {stk:dfc[stk].iloc[-1] for stk in w_goal} - assert len(prices) == len(w_goal) + +def max_allocate(T, df, w_goal): + dfc = df["Close"].unstack(0) + prices = {stk: dfc[stk].iloc[-1] for stk in w_goal} + assert len(prices) == len(w_goal) prices = np.array(list(prices.values())) w_goal = np.array(list(w_goal.values())) p = prices n = len(w_goal) - x = cp.Variable(n,integer=True) + x = cp.Variable(n, integer=True) r = cp.Variable() - #minimize remainder plus norm( target - shares x price) - objective = cp.Minimize(r+cp.norm1(T*w_goal - cp.multiply(x,p))) - #remainder is difference of total and sum(pricesxshares) - strict_remainder = (r+x@p == T) - #no shorting or credit - postive_remainder = r>=0 - constraints = [strict_remainder,postive_remainder] + # minimize remainder plus norm( target - shares x price) + objective = cp.Minimize(r + cp.norm1(T * w_goal - cp.multiply(x, p))) + # remainder is difference of total and sum(pricesxshares) + strict_remainder = r + x @ p == T + # no shorting or credit + postive_remainder = r >= 0 + constraints = [strict_remainder, postive_remainder] prob = cp.Problem(objective, constraints) # The optimal objective value is returned by `prob.solve()`. out = prob.solve() - return {stk:v for stk,v in zip(wght,x.value)} - - - -def make_portfolio(df,gamma=2,risk='ledoit_wolf',returns='mean',rfr=0.045/252,min_weight=0.01,max_weight=None,savefig=True,opt='sharpe',filename=None,name=None,err_factor=1,std_dev=1,allocate_amount=100000): - - pdf = df['Close'].unstack(0) - - if returns=='mean': - mu = pfopt.expected_returns.mean_historical_return(pdf,returns_data=False) + return {stk: v for stk, v in zip(wght, x.value)} + + +def make_portfolio( + df, + gamma=2, + risk="ledoit_wolf", + returns="mean", + rfr=0.045 / 252, + min_weight=0.01, + max_weight=None, + savefig=True, + opt="sharpe", + filename=None, + name=None, + err_factor=1, + std_dev=1, + allocate_amount=100000, +): + + pdf = df["Close"].unstack(0) + + if returns == "mean": + mu = pfopt.expected_returns.mean_historical_return(pdf, returns_data=False) else: - raise KeyError(f'bad return model: {returns}') + raise KeyError(f"bad return model: {returns}") - if risk=='covariance': - S = pfopt.risk_models.sample_cov(pdf,returns_data=False) + if risk == "covariance": + S = pfopt.risk_models.sample_cov(pdf, returns_data=False) # add downside cov semicov - elif risk == 'ledoit_wolf': - S = pfopt.risk_models.CovarianceShrinkage(pdf,returns_data=False).ledoit_wolf() + elif risk == "ledoit_wolf": + S = pfopt.risk_models.CovarianceShrinkage(pdf, returns_data=False).ledoit_wolf() else: - raise KeyError(f'bad risk model: {risk}') + raise KeyError(f"bad risk model: {risk}") Nl = pdf.shape[0] Nd = pdf.isna().sum() - Nt = (Nl-Nd) + Nt = Nl - Nd s = S.to_numpy() - si = np.sum(np.eye(s.shape[0])*s,axis=1) - cycle_frac = (Nt/252)/10 - cycle_penalty = (1/cycle_frac)**2 - - stderr = si*err_factor*cycle_penalty/np.sqrt(Nt) - safelim = si*std_dev + si = np.sum(np.eye(s.shape[0]) * s, axis=1) + cycle_frac = (Nt / 252) / 10 + cycle_penalty = (1 / cycle_frac) ** 2 + + stderr = si * err_factor * cycle_penalty / np.sqrt(Nt) + safelim = si * std_dev mu_base = mu - #print(cycle_penalty,stderr,safelim) + # print(cycle_penalty,stderr,safelim) if err_factor != 0: - mu = np.maximum(mu - stderr,0) + mu = np.maximum(mu - stderr, 0) if std_dev != 0: - mu = np.maximum(mu - safelim,0) + mu = np.maximum(mu - safelim, 0) # Optimize for maximal Sharpe ratio ef = pfopt.EfficientFrontier(mu, S) ef.add_objective(pfopt.objective_functions.L2_reg, gamma=gamma) - #if min_weight: + # if min_weight: # ef.add_constraint(lambda x: x>=min_weight) if max_weight: - ef.add_constraint(lambda x: x<=max_weight) + ef.add_constraint(lambda x: x <= max_weight) - if opt == 'sharpe': + if opt == "sharpe": weights = ef.max_sharpe(rfr) - elif opt == 'min_volatility': + elif opt == "min_volatility": weights = ef.min_volatility() - elif opt == 'eff_return': - weights = ef.efficient_return(0.9*mu.max()+0.1*mu.min()) - elif opt == 'eff_risk': - weights = ef.efficient_risk(0.9*S.min()+0.1*S.max()) - + elif opt == "eff_return": + weights = ef.efficient_return(0.9 * mu.max() + 0.1 * mu.min()) + elif opt == "eff_risk": + weights = ef.efficient_risk(0.9 * S.min() + 0.1 * S.max()) - perf = ef.portfolio_performance(True,risk_free_rate=rfr) + perf = ef.portfolio_performance(True, risk_free_rate=rfr) warray = np.array(list(weights.values())) mu_act = np.sum(mu_base * warray) - wfilt = {k:v for k,v in weights.items() if v >= min_weight and not pdf[k].isna().iloc[-1]} + wfilt = {k: v for k, v in weights.items() if v >= min_weight and not pdf[k].isna().iloc[-1]} tot = np.sum(list(wfilt.values())) - wfilt = {k:v/tot for k,v in wfilt.items()} + wfilt = {k: v / tot for k, v in wfilt.items()} shares = None if allocate_amount: - shares = simple_allocate(allocate_amount,df,wfilt) + shares = simple_allocate(allocate_amount, df, wfilt) + + fig, ax = plot_portfolio(wfilt, shares) - fig,ax = plot_portfolio(wfilt,shares) - if name is None: - name = randomname.get_name(adj=('algorithms','temperature','corporate_prefixes','quantity'),noun=('accounting','corporate','algorithms')) + name = randomname.get_name( + adj=("algorithms", "temperature", "corporate_prefixes", "quantity"), + noun=("accounting", "corporate", "algorithms"), + ) if filename is None: - filename = f'Portfolio_{name}_{opt}_{returns}_{risk}' - - ax.set_title(f'{name} portfolio| risk:{risk} return:{returns} opt:{opt} gamma:{gamma}\n Allocate: ${allocate_amount} Min Return: {perf[0]*100.:3.2f} Act Return: {mu_act*100:3.2f}% Ann Volatility: {perf[1]*100.:3.2f}') + filename = f"Portfolio_{name}_{opt}_{returns}_{risk}" + + ax.set_title( + f"{name} portfolio| risk:{risk} return:{returns} opt:{opt} gamma:{gamma}\n Allocate: ${allocate_amount} Min Return: {perf[0]*100.:3.2f} Act Return: {mu_act*100:3.2f}% Ann Volatility: {perf[1]*100.:3.2f}" + ) if savefig and data.media_dir: - pth = os.path.join(data.media_dir,filename) + pth = os.path.join(data.media_dir, filename) fig.savefig(pth) return wfilt + def cli(): - parser = argparse.ArgumentParser('Portfolio Generator') - parser.add_argument('-risk',choices=['covariance','ledoit_wolf'],default='ledoit_wolf',help='select the risk model, standard covariance or the extremity filtering `ledoit wolf` model') - parser.add_argument('-retrn',choices=['mean'],default='mean',help='return model - mean historical performance averages') - parser.add_argument('-opt',choices=['sharpe','min_volatility','eff_return','eff_risk'],default='sharpe',help='optimization model: maximum risk to return model via sharpe, min_volatility only considers risk, and efficient models will try to achieve 90 percent of the best performing asset') - parser.add_argument('-cls',choices=['etfs','stocks','all'],default='all',help='choose which type of items to trade') - parser.add_argument('-alloc',type=int,default=None,help='choose the amount of money to allocate, this will label the output chart with the number of shares to purchase') - parser.add_argument('-name',type=str,default=None,help='add a name to the portfolio, if none provided a randomly generated name will be created') - parser.add_argument('-filename',type=str,default=None,help='where to store the file, by default it will be stored in a dir set by `TRADEMAN_MEDIA_DIR` ') - parser.add_argument('-rfr',type=float,default=0.045,help='the risk free rate, adjusted per daily returns') - parser.add_argument('-gamma',type=float,default=1,help='the weight regularizer, large values penalize small weight values, make 0 to not penalize small weights') - parser.add_argument('-cycl-err',type=float,default=0,help='default: 0| penalize new assets returns by a factor of economic cycle: `cycle-err x standard error x (10/Nyears)^2`') - parser.add_argument('-std-err',type=float,default=0,help='default: 0| penalize returns by subtracting the `std-err x std-dev`') - - parser.add_argument('-min-wght',type=float,default=0.01,help='assets less than this percent are filtered from the final portfolio') - parser.add_argument('-max-wght',type=float,default = None,help='assets are limited to this max percentage') - parser.add_argument('-in','--include',type=str,default=None,help='csv of a strict include on the ticker name') - parser.add_argument('-ex','--exclude',type=str,default=None,help='csv of a strict exclude on the ticker name') + parser = argparse.ArgumentParser("Portfolio Generator") + parser.add_argument( + "-risk", + choices=["covariance", "ledoit_wolf"], + default="ledoit_wolf", + help="select the risk model, standard covariance or the extremity filtering `ledoit wolf` model", + ) + parser.add_argument( + "-retrn", + choices=["mean"], + default="mean", + help="return model - mean historical performance averages", + ) + parser.add_argument( + "-opt", + choices=["sharpe", "min_volatility", "eff_return", "eff_risk"], + default="sharpe", + help="optimization model: maximum risk to return model via sharpe, min_volatility only considers risk, and efficient models will try to achieve 90 percent of the best performing asset", + ) + parser.add_argument( + "-cls", + choices=["etfs", "stocks", "all"], + default="all", + help="choose which type of items to trade", + ) + parser.add_argument( + "-alloc", + type=int, + default=None, + help="choose the amount of money to allocate, this will label the output chart with the number of shares to purchase", + ) + parser.add_argument( + "-name", + type=str, + default=None, + help="add a name to the portfolio, if none provided a randomly generated name will be created", + ) + parser.add_argument( + "-filename", + type=str, + default=None, + help="where to store the file, by default it will be stored in a dir set by `TRADEMAN_MEDIA_DIR` ", + ) + parser.add_argument( + "-rfr", type=float, default=0.045, help="the risk free rate, adjusted per daily returns" + ) + parser.add_argument( + "-gamma", + type=float, + default=1, + help="the weight regularizer, large values penalize small weight values, make 0 to not penalize small weights", + ) + parser.add_argument( + "-cycl-err", + type=float, + default=0, + help="default: 0| penalize new assets returns by a factor of economic cycle: `cycle-err x standard error x (10/Nyears)^2`", + ) + parser.add_argument( + "-std-err", + type=float, + default=0, + help="default: 0| penalize returns by subtracting the `std-err x std-dev`", + ) + + parser.add_argument( + "-min-wght", + type=float, + default=0.01, + help="assets less than this percent are filtered from the final portfolio", + ) + parser.add_argument( + "-max-wght", type=float, default=None, help="assets are limited to this max percentage" + ) + parser.add_argument( + "-in", + "--include", + type=str, + default=None, + help="csv of a strict include on the ticker name", + ) + parser.add_argument( + "-ex", + "--exclude", + type=str, + default=None, + help="csv of a strict exclude on the ticker name", + ) args = parser.parse_args() filt = None - inc=None - exc=None + inc = None + exc = None if args.include or args.exclude: - inc = args.include.split(',') if args.include else None - exc = args.exclude.split(',') if args.exclude else None - #print(inc,exc) + inc = args.include.split(",") if args.include else None + exc = args.exclude.split(",") if args.exclude else None + + # print(inc,exc) def filt(item): if inc is not None and item not in inc: return False if exc is not None and item in exc: return False return True - - if args.cls == 'all': - items = data.etfs['Symbol'].to_list()+data.snp500['Symbol'].to_list() + + if args.cls == "all": + items = data.etfs["Symbol"].to_list() + data.snp500["Symbol"].to_list() items = list(set(data.db_existing_symbols()).union(set(items))) - elif args.cls == 'etfs': - items = data.etfs['Symbol'].to_list() - elif args.cls == 'stocks': - items = data.snp500['Symbol'].to_list() + elif args.cls == "etfs": + items = data.etfs["Symbol"].to_list() + elif args.cls == "stocks": + items = data.snp500["Symbol"].to_list() # add included items not in symbol list if inc: for intrd in inc: if intrd not in items: items.append(intrd) - - df = get_items( items,filt) - out = make_portfolio(df,rfr=args.rfr/252,returns=args.retrn,opt=args.opt,risk=args.risk,gamma=args.gamma,allocate_amount=args.alloc,filename=args.filename,name=args.name,min_weight=args.min_wght,max_weight=args.max_wght,err_factor=args.cycl_err,std_dev=args.std_err) + df = get_items(items, filt) + + out = make_portfolio( + df, + rfr=args.rfr / 252, + returns=args.retrn, + opt=args.opt, + risk=args.risk, + gamma=args.gamma, + allocate_amount=args.alloc, + filename=args.filename, + name=args.name, + min_weight=args.min_wght, + max_weight=args.max_wght, + err_factor=args.cycl_err, + std_dev=args.std_err, + ) show() return out +if __name__ == "__main__": -if __name__ == '__main__': - - wght = cli() \ No newline at end of file + wght = cli() From 5688ce9a2ce18f8e90429ca8d75d586d211a757f Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Tue, 9 Sep 2025 17:06:02 -0400 Subject: [PATCH 08/12] Fix formatting in build job dependencies in GitHub Actions workflow --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f333835..12d80bd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -111,7 +111,7 @@ jobs: build: - needs: check-version + needs: [check-version] runs-on: ubuntu-latest strategy: matrix: From 36ac6b36d78778dedb08213f985e29615dde413d Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Tue, 9 Sep 2025 17:06:11 -0400 Subject: [PATCH 09/12] Refactor code formatting and improve consistency across multiple files --- tests/test_trademan.py | 17 ++- trademan/__init__.py | 6 +- trademan/data.py | 131 +++++++++-------- trademan/portfolio.py | 317 +++++++++++++++++++++++++++-------------- 4 files changed, 294 insertions(+), 177 deletions(-) diff --git a/tests/test_trademan.py b/tests/test_trademan.py index abc42fb..c425efa 100644 --- a/tests/test_trademan.py +++ b/tests/test_trademan.py @@ -11,14 +11,19 @@ import subprocess as subp + class TestTrademan(unittest.TestCase): """Tests for `trademan` package.""" def test_market_dl(self): """gets appl""" - dl = data.get_tickers('AAPL') - perf = data.data_db[f'perf/AAPL'] - - def test_cli(self) : - os.system("""trademan -cls etfs -gamma 1 -alloc 100000 -in QQQ,SCHG,VGT,SLV,VIG,SPY,VOO,VUG,IAU,PAVE,NANC""") - os.system("""trademan -cls etfs -gamma 20 -alloc 100000 -in QQQ,SCHG,VGT,SLV,VIG,SPY,VOO,VUG,IAU,PAVE,NANC,KRUZ -cycl-err 100""") \ No newline at end of file + dl = data.get_tickers("AAPL") + perf = data.data_db[f"perf/AAPL"] + + def test_cli(self): + os.system( + """trademan -cls etfs -gamma 1 -alloc 100000 -in QQQ,SCHG,VGT,SLV,VIG,SPY,VOO,VUG,IAU,PAVE,NANC""" + ) + os.system( + """trademan -cls etfs -gamma 20 -alloc 100000 -in QQQ,SCHG,VGT,SLV,VIG,SPY,VOO,VUG,IAU,PAVE,NANC,KRUZ -cycl-err 100""" + ) diff --git a/trademan/__init__.py b/trademan/__init__.py index bea3e5f..ca5555f 100644 --- a/trademan/__init__.py +++ b/trademan/__init__.py @@ -1,10 +1,10 @@ """Top-level package for trademan.""" __author__ = """Kevin Russell""" -__email__ = 'kevin@ottermatics.com' -__version__ = '0.1.0' +__email__ = "kevin@ottermatics.com" +__version__ = "0.1.0" from .data import get_tickers, get_ticker_perf from .portfolio import make_portfolio, plot_portfolio -__all__ = ['get_tickers', 'get_ticker_perf', 'make_portfolio', 'plot_portfolio'] +__all__ = ["get_tickers", "get_ticker_perf", "make_portfolio", "plot_portfolio"] diff --git a/trademan/data.py b/trademan/data.py index f15cf54..565fb81 100644 --- a/trademan/data.py +++ b/trademan/data.py @@ -1,130 +1,135 @@ from shutil import ExecError import diskcache -import os,pathlib +import os, pathlib import pandas as pd import yfinance as yf import numpy as np -import datetime,pytz +import datetime, pytz import logging -import time,random +import time, random import tempfile -FORMAT = '%(asctime)s %(message)s' -logging.basicConfig(level=20,format=FORMAT) -log = logging.getLogger('trade-data') +FORMAT = "%(asctime)s %(message)s" +logging.basicConfig(level=20, format=FORMAT) +log = logging.getLogger("trade-data") this_dir = pathlib.Path(__file__).parent -temp_dir = os.path.join(tempfile.gettempdir(),'trademan') +temp_dir = os.path.join(tempfile.gettempdir(), "trademan") -data_dir_dflt = os.path.join(temp_dir,'data') -data_dir = os.environ.get('TRADEMAN_DATA_DIR',data_dir_dflt) +data_dir_dflt = os.path.join(temp_dir, "data") +data_dir = os.environ.get("TRADEMAN_DATA_DIR", data_dir_dflt) -db_path = os.path.join(data_dir,'assets.db') +db_path = os.path.join(data_dir, "assets.db") data_db = diskcache.Cache(db_path) -media_dir_dflt = os.path.join(temp_dir,'media') -media_dir = os.environ.get('TRADEMAN_MEDIA_DIR',media_dir_dflt) +media_dir_dflt = os.path.join(temp_dir, "media") +media_dir = os.environ.get("TRADEMAN_MEDIA_DIR", media_dir_dflt) -print(f'TRADEMAN DATA: {data_dir}') -print(f'TRADEMAN MEDIA: {media_dir}') -print(f'use `TRADEMAN_MEDIA_DIR` or `TRADEMAN_DATA_DIR` envvars to customize paths') -os.makedirs(data_dir,exist_ok=True) -os.makedirs(media_dir,exist_ok=True) +print(f"TRADEMAN DATA: {data_dir}") +print(f"TRADEMAN MEDIA: {media_dir}") +print(f"use `TRADEMAN_MEDIA_DIR` or `TRADEMAN_DATA_DIR` envvars to customize paths") +os.makedirs(data_dir, exist_ok=True) +os.makedirs(media_dir, exist_ok=True) -ticker_record = 'tickers' +ticker_record = "tickers" -dl_keys = {'perf':lambda x: x.history(period='max',interval='1d'), - 'info':lambda x: x.info} +dl_keys = {"perf": lambda x: x.history(period="max", interval="1d"), "info": lambda x: x.info} -db_set_keys = {'perf':{'expire':3600*24*5}, - 'info':{'expire':3600*24*30}} +db_set_keys = {"perf": {"expire": 3600 * 24 * 5}, "info": {"expire": 3600 * 24 * 30}} + +all_stocks = pd.read_csv(os.path.join(this_dir, "stock_info.csv")) +snp500 = pd.read_csv(os.path.join(this_dir, "snp500_members.csv")) +etfs = pd.read_csv(os.path.join(this_dir, "etfs.csv")) -all_stocks = pd.read_csv(os.path.join(this_dir,'stock_info.csv')) -snp500 = pd.read_csv(os.path.join(this_dir,'snp500_members.csv')) -etfs = pd.read_csv(os.path.join(this_dir,'etfs.csv')) def db_existing_symbols(): - return [s.replace('tickers/','') for s in data_db.iterkeys() if s.startswith('tickers/')] + return [s.replace("tickers/", "") for s in data_db.iterkeys() if s.startswith("tickers/")] + -def _dl(ticker,select=None,delay=5): +def _dl(ticker, select=None, delay=5): """smartly accesses diskcache data if it exists, else download it""" out = {} - failures = data_db.get(f'ticker_failures',[]) + failures = data_db.get(f"ticker_failures", []) if ticker in failures: - log.info(f'skipping previously failed {ticker}') + log.info(f"skipping previously failed {ticker}") return - + try: yfobj = yf.Ticker(ticker) - for dk,func in dl_keys.items(): - - key = f'{dk}/{ticker.lower()}' + for dk, func in dl_keys.items(): + + key = f"{dk}/{ticker.lower()}" if key in data_db: - log.debug(f'db get: {ticker}') + log.debug(f"db get: {ticker}") dlip = data_db[key] else: - dlwait = delay*(1+random.random()/2) - log.info(f'downloading {dk}/{ticker} after: {dlwait}s') + dlwait = delay * (1 + random.random() / 2) + log.info(f"downloading {dk}/{ticker} after: {dlwait}s") time.sleep(dlwait) dlip = func(yfobj) - set_kw = {'tag':dk,**db_set_keys.get(dk,{})} - data_db.set(key,dlip,**set_kw) + set_kw = {"tag": dk, **db_set_keys.get(dk, {})} + data_db.set(key, dlip, **set_kw) - #add to record: + # add to record: get_time = datetime.datetime.now(tz=pytz.utc).isoformat() - data_db.set(f'{ticker_record}/{ticker}', get_time,tag='record') + data_db.set(f"{ticker_record}/{ticker}", get_time, tag="record") if select is None or dk in select: out[dk] = dlip - return out - + return out + except Exception as e: - log.error(e,msg=f'issue getting: {ticker}') + log.error(e, msg=f"issue getting: {ticker}") failures.append(ticker) - data_db[f'ticker_failures'] = failures + data_db[f"ticker_failures"] = failures + def clear_failures(): - data_db[f'ticker_failures'] = [] + data_db[f"ticker_failures"] = [] + def get_ticker_perf(ticker): dat = _dl(ticker) if dat is None: return - dfms = pd.DataFrame(dat['perf']) - - dfms['perf'] = dc = (dfms['Close'] - dfms['Open'])/(dfms['Open']) - dfms['grwth'] = np.cumprod(1+dc)-1 - dfms['highQlow'] = ec = (dfms['High'] - dfms['Low'])/(dfms['High'] + dfms['Low']) + dfms = pd.DataFrame(dat["perf"]) + + dfms["perf"] = dc = (dfms["Close"] - dfms["Open"]) / (dfms["Open"]) + dfms["grwth"] = np.cumprod(1 + dc) - 1 + dfms["highQlow"] = ec = (dfms["High"] - dfms["Low"]) / (dfms["High"] + dfms["Low"]) return dfms + def get_tickers(tickers): - - if isinstance(tickers,str): - tickers =tickers.split(',') - log.info(f'getting {len(tickers)}| {str(list(tickers))[:100]}') + if isinstance(tickers, str): + tickers = tickers.split(",") - data ={} + log.info(f"getting {len(tickers)}| {str(list(tickers))[:100]}") + + data = {} for tick in tickers: dfms = get_ticker_perf(tick) if dfms is not None: data[tick] = dfms - table = pd.concat(data.values(),keys=data.keys()) + table = pd.concat(data.values(), keys=data.keys()) + + return table - return table def main(clear=False): - if clear: + if clear: clear_failures() - #TODO: clear db ect via options - get_tickers(etfs['Symbol']) - get_tickers(snp500['Symbol']) + # TODO: clear db ect via options + get_tickers(etfs["Symbol"]) + get_tickers(snp500["Symbol"]) + -if __name__ == '__main__': - main() \ No newline at end of file +if __name__ == "__main__": + main() diff --git a/trademan/portfolio.py b/trademan/portfolio.py index 30f08ef..3348ca9 100644 --- a/trademan/portfolio.py +++ b/trademan/portfolio.py @@ -8,17 +8,18 @@ import cvxpy as cp import argparse -def get_items(canidates,filtr:list=None): - + +def get_items(canidates, filtr: list = None): + tickers = canidates if filtr: - tickers = list(filter(filtr,canidates)) - print(f'filtered: {set(canidates).difference(set(tickers))}') + tickers = list(filter(filtr, canidates)) + print(f"filtered: {set(canidates).difference(set(tickers))}") df = data.get_tickers(tickers) return df - -def plot_portfolio(weights,shares=None): + +def plot_portfolio(weights, shares=None): """ Plot the portfolio weights as a horizontal bar chart @@ -29,7 +30,7 @@ def plot_portfolio(weights,shares=None): :return: matplotlib axis :rtype: matplotlib.axes """ - fig,ax = subplots(figsize=(6,10)) + fig, ax = subplots(figsize=(6, 10)) desc = sorted(weights.items(), key=lambda x: x[1], reverse=True) labels = [i[0] for i in desc] @@ -39,194 +40,300 @@ def plot_portfolio(weights,shares=None): hbars = ax.barh(y_pos, vals) if shares: - ax.bar_label(hbars,labels=[f'{v}' for v in shares.values()]) + ax.bar_label(hbars, labels=[f"{v}" for v in shares.values()]) ax.set_xlabel("Weight") ax.set_yticks(y_pos) ax.set_yticklabels(labels) ax.invert_yaxis() - return fig,ax + return fig, ax + -def simple_allocate(T,df,w_goal): - dfc = df['Close'].unstack(0) +def simple_allocate(T, df, w_goal): + dfc = df["Close"].unstack(0) prices = dfc[list(w_goal)].iloc[-1] - alloc = pfopt.discrete_allocation.DiscreteAllocation(w_goal, prices,T) + alloc = pfopt.discrete_allocation.DiscreteAllocation(w_goal, prices, T) return alloc.greedy_portfolio(True)[0] -def max_allocate(T,df,w_goal): - dfc = df['Close'].unstack(0) - prices = {stk:dfc[stk].iloc[-1] for stk in w_goal} - assert len(prices) == len(w_goal) + +def max_allocate(T, df, w_goal): + dfc = df["Close"].unstack(0) + prices = {stk: dfc[stk].iloc[-1] for stk in w_goal} + assert len(prices) == len(w_goal) prices = np.array(list(prices.values())) w_goal = np.array(list(w_goal.values())) p = prices n = len(w_goal) - x = cp.Variable(n,integer=True) + x = cp.Variable(n, integer=True) r = cp.Variable() - #minimize remainder plus norm( target - shares x price) - objective = cp.Minimize(r+cp.norm1(T*w_goal - cp.multiply(x,p))) - #remainder is difference of total and sum(pricesxshares) - strict_remainder = (r+x@p == T) - #no shorting or credit - postive_remainder = r>=0 - constraints = [strict_remainder,postive_remainder] + # minimize remainder plus norm( target - shares x price) + objective = cp.Minimize(r + cp.norm1(T * w_goal - cp.multiply(x, p))) + # remainder is difference of total and sum(pricesxshares) + strict_remainder = r + x @ p == T + # no shorting or credit + postive_remainder = r >= 0 + constraints = [strict_remainder, postive_remainder] prob = cp.Problem(objective, constraints) # The optimal objective value is returned by `prob.solve()`. out = prob.solve() - return {stk:v for stk,v in zip(wght,x.value)} - - - -def make_portfolio(df,gamma=2,risk='ledoit_wolf',returns='mean',rfr=0.045/252,min_weight=0.01,max_weight=None,savefig=True,opt='sharpe',filename=None,name=None,err_factor=1,std_dev=1,allocate_amount=100000): - - pdf = df['Close'].unstack(0) - - if returns=='mean': - mu = pfopt.expected_returns.mean_historical_return(pdf,returns_data=False) + return {stk: v for stk, v in zip(wght, x.value)} + + +def make_portfolio( + df, + gamma=2, + risk="ledoit_wolf", + returns="mean", + rfr=0.045 / 252, + min_weight=0.01, + max_weight=None, + savefig=True, + opt="sharpe", + filename=None, + name=None, + err_factor=1, + std_dev=1, + allocate_amount=100000, +): + + pdf = df["Close"].unstack(0) + + if returns == "mean": + mu = pfopt.expected_returns.mean_historical_return(pdf, returns_data=False) else: - raise KeyError(f'bad return model: {returns}') + raise KeyError(f"bad return model: {returns}") - if risk=='covariance': - S = pfopt.risk_models.sample_cov(pdf,returns_data=False) + if risk == "covariance": + S = pfopt.risk_models.sample_cov(pdf, returns_data=False) # add downside cov semicov - elif risk == 'ledoit_wolf': - S = pfopt.risk_models.CovarianceShrinkage(pdf,returns_data=False).ledoit_wolf() + elif risk == "ledoit_wolf": + S = pfopt.risk_models.CovarianceShrinkage(pdf, returns_data=False).ledoit_wolf() else: - raise KeyError(f'bad risk model: {risk}') + raise KeyError(f"bad risk model: {risk}") Nl = pdf.shape[0] Nd = pdf.isna().sum() - Nt = (Nl-Nd) + Nt = Nl - Nd s = S.to_numpy() - si = np.sum(np.eye(s.shape[0])*s,axis=1) - cycle_frac = (Nt/252)/10 - cycle_penalty = (1/cycle_frac)**2 - - stderr = si*err_factor*cycle_penalty/np.sqrt(Nt) - safelim = si*std_dev + si = np.sum(np.eye(s.shape[0]) * s, axis=1) + cycle_frac = (Nt / 252) / 10 + cycle_penalty = (1 / cycle_frac) ** 2 + + stderr = si * err_factor * cycle_penalty / np.sqrt(Nt) + safelim = si * std_dev mu_base = mu - #print(cycle_penalty,stderr,safelim) + # print(cycle_penalty,stderr,safelim) if err_factor != 0: - mu = np.maximum(mu - stderr,0) + mu = np.maximum(mu - stderr, 0) if std_dev != 0: - mu = np.maximum(mu - safelim,0) + mu = np.maximum(mu - safelim, 0) # Optimize for maximal Sharpe ratio ef = pfopt.EfficientFrontier(mu, S) ef.add_objective(pfopt.objective_functions.L2_reg, gamma=gamma) - #if min_weight: + # if min_weight: # ef.add_constraint(lambda x: x>=min_weight) if max_weight: - ef.add_constraint(lambda x: x<=max_weight) + ef.add_constraint(lambda x: x <= max_weight) - if opt == 'sharpe': + if opt == "sharpe": weights = ef.max_sharpe(rfr) - elif opt == 'min_volatility': + elif opt == "min_volatility": weights = ef.min_volatility() - elif opt == 'eff_return': - weights = ef.efficient_return(0.9*mu.max()+0.1*mu.min()) - elif opt == 'eff_risk': - weights = ef.efficient_risk(0.9*S.min()+0.1*S.max()) - + elif opt == "eff_return": + weights = ef.efficient_return(0.9 * mu.max() + 0.1 * mu.min()) + elif opt == "eff_risk": + weights = ef.efficient_risk(0.9 * S.min() + 0.1 * S.max()) - perf = ef.portfolio_performance(True,risk_free_rate=rfr) + perf = ef.portfolio_performance(True, risk_free_rate=rfr) warray = np.array(list(weights.values())) mu_act = np.sum(mu_base * warray) - wfilt = {k:v for k,v in weights.items() if v >= min_weight and not pdf[k].isna().iloc[-1]} + wfilt = {k: v for k, v in weights.items() if v >= min_weight and not pdf[k].isna().iloc[-1]} tot = np.sum(list(wfilt.values())) - wfilt = {k:v/tot for k,v in wfilt.items()} + wfilt = {k: v / tot for k, v in wfilt.items()} shares = None if allocate_amount: - shares = simple_allocate(allocate_amount,df,wfilt) + shares = simple_allocate(allocate_amount, df, wfilt) + + fig, ax = plot_portfolio(wfilt, shares) - fig,ax = plot_portfolio(wfilt,shares) - if name is None: - name = randomname.get_name(adj=('algorithms','temperature','corporate_prefixes','quantity'),noun=('accounting','corporate','algorithms')) + name = randomname.get_name( + adj=("algorithms", "temperature", "corporate_prefixes", "quantity"), + noun=("accounting", "corporate", "algorithms"), + ) if filename is None: - filename = f'Portfolio_{name}_{opt}_{returns}_{risk}' - - ax.set_title(f'{name} portfolio| risk:{risk} return:{returns} opt:{opt} gamma:{gamma}\n Allocate: ${allocate_amount} Min Return: {perf[0]*100.:3.2f} Act Return: {mu_act*100:3.2f}% Ann Volatility: {perf[1]*100.:3.2f}') + filename = f"Portfolio_{name}_{opt}_{returns}_{risk}" + + ax.set_title( + f"{name} portfolio| risk:{risk} return:{returns} opt:{opt} gamma:{gamma}\n Allocate: ${allocate_amount} Min Return: {perf[0]*100.:3.2f} Act Return: {mu_act*100:3.2f}% Ann Volatility: {perf[1]*100.:3.2f}" + ) if savefig and data.media_dir: - pth = os.path.join(data.media_dir,filename) + pth = os.path.join(data.media_dir, filename) fig.savefig(pth) return wfilt + def cli(): - parser = argparse.ArgumentParser('Portfolio Generator') - parser.add_argument('-risk',choices=['covariance','ledoit_wolf'],default='ledoit_wolf',help='select the risk model, standard covariance or the extremity filtering `ledoit wolf` model') - parser.add_argument('-retrn',choices=['mean'],default='mean',help='return model - mean historical performance averages') - parser.add_argument('-opt',choices=['sharpe','min_volatility','eff_return','eff_risk'],default='sharpe',help='optimization model: maximum risk to return model via sharpe, min_volatility only considers risk, and efficient models will try to achieve 90 percent of the best performing asset') - parser.add_argument('-cls',choices=['etfs','stocks','all'],default='all',help='choose which type of items to trade') - parser.add_argument('-alloc',type=int,default=None,help='choose the amount of money to allocate, this will label the output chart with the number of shares to purchase') - parser.add_argument('-name',type=str,default=None,help='add a name to the portfolio, if none provided a randomly generated name will be created') - parser.add_argument('-filename',type=str,default=None,help='where to store the file, by default it will be stored in a dir set by `TRADEMAN_MEDIA_DIR` ') - parser.add_argument('-rfr',type=float,default=0.045,help='the risk free rate, adjusted per daily returns') - parser.add_argument('-gamma',type=float,default=1,help='the weight regularizer, large values penalize small weight values, make 0 to not penalize small weights') - parser.add_argument('-cycl-err',type=float,default=0,help='default: 0| penalize new assets returns by a factor of economic cycle: `cycle-err x standard error x (10/Nyears)^2`') - parser.add_argument('-std-err',type=float,default=0,help='default: 0| penalize returns by subtracting the `std-err x std-dev`') - - parser.add_argument('-min-wght',type=float,default=0.01,help='assets less than this percent are filtered from the final portfolio') - parser.add_argument('-max-wght',type=float,default = None,help='assets are limited to this max percentage') - parser.add_argument('-in','--include',type=str,default=None,help='csv of a strict include on the ticker name') - parser.add_argument('-ex','--exclude',type=str,default=None,help='csv of a strict exclude on the ticker name') + parser = argparse.ArgumentParser("Portfolio Generator") + parser.add_argument( + "-risk", + choices=["covariance", "ledoit_wolf"], + default="ledoit_wolf", + help="select the risk model, standard covariance or the extremity filtering `ledoit wolf` model", + ) + parser.add_argument( + "-retrn", + choices=["mean"], + default="mean", + help="return model - mean historical performance averages", + ) + parser.add_argument( + "-opt", + choices=["sharpe", "min_volatility", "eff_return", "eff_risk"], + default="sharpe", + help="optimization model: maximum risk to return model via sharpe, min_volatility only considers risk, and efficient models will try to achieve 90 percent of the best performing asset", + ) + parser.add_argument( + "-cls", + choices=["etfs", "stocks", "all"], + default="all", + help="choose which type of items to trade", + ) + parser.add_argument( + "-alloc", + type=int, + default=None, + help="choose the amount of money to allocate, this will label the output chart with the number of shares to purchase", + ) + parser.add_argument( + "-name", + type=str, + default=None, + help="add a name to the portfolio, if none provided a randomly generated name will be created", + ) + parser.add_argument( + "-filename", + type=str, + default=None, + help="where to store the file, by default it will be stored in a dir set by `TRADEMAN_MEDIA_DIR` ", + ) + parser.add_argument( + "-rfr", type=float, default=0.045, help="the risk free rate, adjusted per daily returns" + ) + parser.add_argument( + "-gamma", + type=float, + default=1, + help="the weight regularizer, large values penalize small weight values, make 0 to not penalize small weights", + ) + parser.add_argument( + "-cycl-err", + type=float, + default=0, + help="default: 0| penalize new assets returns by a factor of economic cycle: `cycle-err x standard error x (10/Nyears)^2`", + ) + parser.add_argument( + "-std-err", + type=float, + default=0, + help="default: 0| penalize returns by subtracting the `std-err x std-dev`", + ) + + parser.add_argument( + "-min-wght", + type=float, + default=0.01, + help="assets less than this percent are filtered from the final portfolio", + ) + parser.add_argument( + "-max-wght", type=float, default=None, help="assets are limited to this max percentage" + ) + parser.add_argument( + "-in", + "--include", + type=str, + default=None, + help="csv of a strict include on the ticker name", + ) + parser.add_argument( + "-ex", + "--exclude", + type=str, + default=None, + help="csv of a strict exclude on the ticker name", + ) args = parser.parse_args() filt = None - inc=None - exc=None + inc = None + exc = None if args.include or args.exclude: - inc = args.include.split(',') if args.include else None - exc = args.exclude.split(',') if args.exclude else None - #print(inc,exc) + inc = args.include.split(",") if args.include else None + exc = args.exclude.split(",") if args.exclude else None + + # print(inc,exc) def filt(item): if inc is not None and item not in inc: return False if exc is not None and item in exc: return False return True - - if args.cls == 'all': - items = data.etfs['Symbol'].to_list()+data.snp500['Symbol'].to_list() + + if args.cls == "all": + items = data.etfs["Symbol"].to_list() + data.snp500["Symbol"].to_list() items = list(set(data.db_existing_symbols()).union(set(items))) - elif args.cls == 'etfs': - items = data.etfs['Symbol'].to_list() - elif args.cls == 'stocks': - items = data.snp500['Symbol'].to_list() + elif args.cls == "etfs": + items = data.etfs["Symbol"].to_list() + elif args.cls == "stocks": + items = data.snp500["Symbol"].to_list() # add included items not in symbol list if inc: for intrd in inc: if intrd not in items: items.append(intrd) - - df = get_items( items,filt) - out = make_portfolio(df,rfr=args.rfr/252,returns=args.retrn,opt=args.opt,risk=args.risk,gamma=args.gamma,allocate_amount=args.alloc,filename=args.filename,name=args.name,min_weight=args.min_wght,max_weight=args.max_wght,err_factor=args.cycl_err,std_dev=args.std_err) + df = get_items(items, filt) + + out = make_portfolio( + df, + rfr=args.rfr / 252, + returns=args.retrn, + opt=args.opt, + risk=args.risk, + gamma=args.gamma, + allocate_amount=args.alloc, + filename=args.filename, + name=args.name, + min_weight=args.min_wght, + max_weight=args.max_wght, + err_factor=args.cycl_err, + std_dev=args.std_err, + ) show() return out +if __name__ == "__main__": -if __name__ == '__main__': - - wght = cli() \ No newline at end of file + wght = cli() From c6c51b8761bb22d1cf9fcc451e8e7abe85877610 Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Tue, 9 Sep 2025 19:16:58 -0400 Subject: [PATCH 10/12] Update Python version matrix in build and publish workflows --- .github/workflows/build.yml | 2 +- .github/workflows/publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 12d80bd..3754fea 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -115,7 +115,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 18f41d2..6b2dd0b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -73,7 +73,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.11"] + python-version: ["3.9","3.10","3.11", "3.12"] steps: - uses: actions/checkout@v4 From 33ca26605d3c107be6c27ef34db4b7794477b23f Mon Sep 17 00:00:00 2001 From: SoundsSerious Date: Tue, 9 Sep 2025 19:37:32 -0400 Subject: [PATCH 11/12] Fix case sensitivity in data retrieval and enhance download delay logic --- tests/test_trademan.py | 2 +- trademan/data.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_trademan.py b/tests/test_trademan.py index c425efa..cf4236e 100644 --- a/tests/test_trademan.py +++ b/tests/test_trademan.py @@ -18,7 +18,7 @@ class TestTrademan(unittest.TestCase): def test_market_dl(self): """gets appl""" dl = data.get_tickers("AAPL") - perf = data.data_db[f"perf/AAPL"] + perf = data.data_db[f"perf/AAPL".lower()] def test_cli(self): os.system( diff --git a/trademan/data.py b/trademan/data.py index 565fb81..1cd9c75 100644 --- a/trademan/data.py +++ b/trademan/data.py @@ -9,6 +9,8 @@ import time, random import tempfile + + FORMAT = "%(asctime)s %(message)s" logging.basicConfig(level=20, format=FORMAT) log = logging.getLogger("trade-data") @@ -51,6 +53,8 @@ def _dl(ticker, select=None, delay=5): """smartly accesses diskcache data if it exists, else download it""" out = {} + cur_count = data_db.incr('dl_counter',default=0) + failures = data_db.get(f"ticker_failures", []) if ticker in failures: log.info(f"skipping previously failed {ticker}") @@ -67,7 +71,10 @@ def _dl(ticker, select=None, delay=5): else: dlwait = delay * (1 + random.random() / 2) log.info(f"downloading {dk}/{ticker} after: {dlwait}s") - time.sleep(dlwait) + + if cur_count > 15: + #first 10 free + time.sleep(dlwait*(1+cur_count*0.1/15)) dlip = func(yfobj) set_kw = {"tag": dk, **db_set_keys.get(dk, {})} From c7f864cc10588c80eb9c7443378d3e7ad31eff38 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Tue, 9 Sep 2025 23:37:48 +0000 Subject: [PATCH 12/12] Auto-format code with Black --- trademan/data.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/trademan/data.py b/trademan/data.py index 1cd9c75..b14b3f3 100644 --- a/trademan/data.py +++ b/trademan/data.py @@ -10,7 +10,6 @@ import tempfile - FORMAT = "%(asctime)s %(message)s" logging.basicConfig(level=20, format=FORMAT) log = logging.getLogger("trade-data") @@ -53,7 +52,7 @@ def _dl(ticker, select=None, delay=5): """smartly accesses diskcache data if it exists, else download it""" out = {} - cur_count = data_db.incr('dl_counter',default=0) + cur_count = data_db.incr("dl_counter", default=0) failures = data_db.get(f"ticker_failures", []) if ticker in failures: @@ -71,10 +70,10 @@ def _dl(ticker, select=None, delay=5): else: dlwait = delay * (1 + random.random() / 2) log.info(f"downloading {dk}/{ticker} after: {dlwait}s") - + if cur_count > 15: - #first 10 free - time.sleep(dlwait*(1+cur_count*0.1/15)) + # first 10 free + time.sleep(dlwait * (1 + cur_count * 0.1 / 15)) dlip = func(yfobj) set_kw = {"tag": dk, **db_set_keys.get(dk, {})}