diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md deleted file mode 100644 index 215b057c..00000000 --- a/.github/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,76 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, sex characteristics, gender identity and expression, -level of experience, education, socio-economic status, nationality, personal -appearance, race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment -include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at lcsb-r3 .at. uni.lu. All -complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. The project team is -obligated to maintain confidentiality with regard to the reporter of an incident. -Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good -faith may face temporary or permanent repercussions as determined by other -members of the project's leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html - -[homepage]: https://www.contributor-covenant.org - -For answers to common questions about this code of conduct, see -https://www.contributor-covenant.org/faq diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md deleted file mode 100644 index 46342d61..00000000 --- a/.github/CONTRIBUTING.md +++ /dev/null @@ -1,34 +0,0 @@ -# Contributing to GigaSOM - -:+1::tada: Thanks for taking the time to contribute to [GigaSOM.jl](https://github.com/LCSB-BioCore/GigaSOM.jl)! :tada::+1: - -## How can I contribute? - -A complete technical guide on how to get started contributing is given [here](https://lcsb-biocore.github.io/GigaSOM.jl/stable/howToContribute/). - -## How to report a bug or suggest an enhancement - -Please use the [GitHub issue tracker](https://github.com/LCSB-BioCore/GigaSOM.jl/issues) to report any problems with the software, and discuss any potential questions about GigaSOM use. - -Before creating bug reports, please check [the open -issues](https://github.com/LCSB-BioCore/GigaSOM.jl/issues) as you might find -out that you don't need to create one. When you are creating a bug report, -please include as many details as possible. Fill out the required template, the -information it asks for helps us resolve issues faster. - -> If you find a Closed issue that seems like it is the same thing that - you're experiencing, open a new issue and include a link to the original issue - in the body of your new one. - -If reporting issues, please do not forget to include a description of conditions under which the issue occurs; preferably a code snippet that can be run separately (sometimes termed "minimal crashing example"). - -## How to submit a pull request (PR)? - -Please follow these steps to have your contribution considered by the maintainers: - -1. Follow all instructions in the template -2. Submit your pull request against the `develop` branch -3. After you submit your pull request, verify that all status checks are passing - -After you submitted a pull request, a label might be assigned that allows us -to track and manage issues and pull requests. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/custom.md b/.github/ISSUE_TEMPLATE/custom.md deleted file mode 100644 index b223dd03..00000000 --- a/.github/ISSUE_TEMPLATE/custom.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -name: General issue (performance or bug) -about: Describe your issue related to performance or a bug -title: '' -labels: '' -assignees: '' - ---- - -**Summary** - -**Environment** - -*Please provide all information on your environment here, including the -version of Julia* - -**Steps to reproduce** - - 1. - 2. - 3. - -**Expected behavior** - -**Actual behavior** - -**Additional information** diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index bbcbbe7d..00000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: '' -labels: '' -assignees: '' - ---- - -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - -**Describe the solution you'd like** -A clear and concise description of what you want to happen. - -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - -**Additional context** -Add any other context or screenshots about the feature request here. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index d91a23a5..00000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,4 +0,0 @@ -# Description of the proposed change - -*Please include a short description of enhancement here* - diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml index 2afd0558..8765dd6e 100644 --- a/.github/workflows/CompatHelper.yml +++ b/.github/workflows/CompatHelper.yml @@ -37,8 +37,7 @@ jobs: - name: "Run CompatHelper" run: | import CompatHelper - CompatHelper.main(master_branch="develop") + CompatHelper.main() shell: julia --color=yes {0} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1268ea37..2b1acb88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ on: push: branches: - master + pull_request: jobs: test: @@ -13,29 +14,22 @@ jobs: fail-fast: false matrix: version: - - 'nightly' + - '1' # This is always the latest stable release in the 1.X series + - '1.10' # LTS + #- 'nightly' os: - ubuntu-latest - - macOS-latest + #- macOS-latest #- windows-latest arch: - x64 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - - uses: actions/cache@v1 - env: - cache-name: cache-artifacts - with: - path: ~/.julia/artifacts - key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} - restore-keys: | - ${{ runner.os }}-test-${{ env.cache-name }}- - ${{ runner.os }}-test- - ${{ runner.os }}- + - uses: julia-actions/cache@v2 - uses: julia-actions/julia-buildpkg@latest - run: | git config --global user.name Tester @@ -43,6 +37,7 @@ jobs: - uses: julia-actions/julia-runtest@latest continue-on-error: ${{ matrix.version == 'nightly' }} - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v1 + - uses: codecov/codecov-action@v4 with: - file: lcov.info + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 33118fa5..20533c09 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,24 +1,40 @@ -name: Publish Docker image + +name: build docker image on: push: - tags: - - "v*" - release: - types: [published, created] + branches: + - master + tags: '*' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: lcsb-biocore/gigasom.jl # lowercase of ${{ github.repository }} jobs: - push_to_registry: - name: Push Docker image to GitHub Packages + build: runs-on: ubuntu-latest + permissions: + contents: read + packages: write steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Push to GitHub Registry - uses: mr-smithers-excellent/docker-build-push@v5 - with: - image: gigasom.jl - tags: latest - registry: ghcr.io - username: cylon-x - password: ${{ secrets.docker_token }} + - name: Checkout + uses: actions/checkout@v4 + - name: log into docker registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: build and push the docker container + uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f8f181b3..e0cab802 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,26 +1,24 @@ + # ref: https://juliadocs.github.io/Documenter.jl/stable/man/hosting/#GitHub-Actions-1 name: Documentation on: + pull_request: push: branches: - - develop - master tags: '*' - release: - types: [published, created] jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest - with: - version: 1.7 + - uses: julia-actions/cache@v2 - name: Install dependencies - run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()' + run: julia --color=yes --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()' - name: Build and deploy env: DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # For authentication with SSH deploy key - run: julia --project=docs/ docs/make.jl + run: julia --color=yes --project=docs/ docs/make.jl diff --git a/.github/workflows/pr-format.yml b/.github/workflows/pr-format.yml new file mode 100644 index 00000000..3ff70d14 --- /dev/null +++ b/.github/workflows/pr-format.yml @@ -0,0 +1,62 @@ +on: + pull_request: + issue_comment: + types: [created] + +name: Formatting + +jobs: + formatting: + if: github.event_name == 'pull_request' || (github.event_name == 'issue_comment' && github.event.issue.pull_request && (github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'OWNER' || github.event.issue.user.id == github.event.comment.user.id) && startsWith(github.event.comment.body, '/format') ) + runs-on: ubuntu-latest + steps: + - name: Clone the repository + uses: actions/checkout@v4 + - name: Checkout the pull request code # this checks out the actual branch so that one can commit into it + if: github.event_name == 'issue_comment' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh pr checkout ${{ github.event.issue.number }} + - name: Install JuliaFormatter and format + run: | + julia --color=yes -e 'import Pkg; Pkg.add("JuliaFormatter")' + julia --color=yes -e 'using JuliaFormatter; format(".")' + - name: Remove trailing whitespace + run: | + find -name '*.jl' -or -name '*.md' -or -name '*.toml' -or -name '*.yml' | while read filename ; do + # remove any trailing spaces + sed --in-place -e 's/\s*$//' "$filename" + # add a final newline if missing + if [[ -s "$filename" && $(tail -c 1 "$filename" |wc -l) -eq 0 ]] ; then + echo >> "$filename" + fi + # squash superfluous final newlines + sed -i -e :a -e '/^\n*$/{$d;N;};/\n$/ba' "$filename" + done + - name: Fail on formatting problems + if: github.event_name == 'pull_request' + run: | + if git diff --exit-code --quiet + then echo "Looks OK" + else echo "Formatting fixes required!"; git diff -p ; exit 1 + fi + - name: Commit fixes + if: github.event_name == 'issue_comment' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [ `git status -s | wc -l` -ne 0 ] ; then + git config --local user.name "$GITHUB_ACTOR" + git config --local user.email "$GITHUB_ACTOR@users.noreply.github.com" + git commit -a -m "automatic formatting" -m "triggered by @$GITHUB_ACTOR on PR #${{ github.event.issue.number }}" + if git push + then gh pr comment ${{ github.event.issue.number }} --body \ + ":heavy_check_mark: Auto-formatting triggered by [this comment](${{ github.event.comment.html_url }}) succeeded, commited as `git rev-parse HEAD`" + else gh pr comment ${{ github.event.issue.number }} --body \ + ":x: Auto-formatting triggered by [this comment](${{ github.event.comment.html_url }}) failed, perhaps someone pushed to the PR in the meantime?" + fi + else + gh pr comment ${{ github.event.issue.number }} --body \ + ":sunny: Auto-formatting triggered by [this comment](${{ github.event.comment.html_url }}) succeeded, but the code was already formatted correctly." + fi diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index f4c4621e..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,186 +0,0 @@ - -stages: - - test - - assets - -variables: - GIT_STRATEGY: clone - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - APPTAINER_DOCKER_TAG: "v3.9.4" - DOCKER_HUB_TAG: "lcsbbiocore/gigasom.jl" - DOCKER_GHCR_TAG: "ghcr.io/lcsb-biocore/docker/gigasom.jl" - APPTAINER_GHCR_TAG: "lcsb-biocore/apptainer/gigasom.jl" - -# -# Predefined conditions for triggering jobs -# - -.global_trigger_pull_request: &global_trigger_pull_request - rules: - - if: $CI_COMMIT_BRANCH == "develop" - when: never - - if: $CI_COMMIT_BRANCH == "master" - when: never - - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" - -.global_trigger_full_tests: &global_trigger_full_tests - rules: - - if: $CI_COMMIT_BRANCH == "develop" - - if: $CI_COMMIT_BRANCH == "master" - -.global_trigger_test_containers: &global_trigger_test_containers - rules: - - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" - when: never - - if: $CI_COMMIT_BRANCH == "develop" - -.global_trigger_release_containers: &global_trigger_release_containers - rules: - - if: $CI_COMMIT_TAG =~ /^v/ - -# -# Test environment & platform settings -# - -.global_dind: &global_dind - image: docker:20.10.12 - tags: - - privileged - services: - - name: docker:20.10.12-dind - command: ["--tls=false", "--mtu=1458", "--registry-mirror", "https://docker-registry.lcsb.uni.lu"] - before_script: - - docker login -u $CI_USER_NAME -p $GITLAB_ACCESS_TOKEN $CI_REGISTRY - -.global_julia18: &global_julia18 - variables: - JULIA_VER: "v1.8.2" - -.global_env_linux: &global_env_linux - script: - - $ARTENOLIS_SOFT_PATH/julia/$JULIA_VER/bin/julia --inline=yes --check-bounds=yes --color=yes --project=@. -e 'import Pkg; Pkg.test(; coverage = true)' - -.global_env_win: &global_env_win - script: - - $global:LASTEXITCODE = 0 # Note the global prefix. - - Invoke-Expression $Env:ARTENOLIS_SOFT_PATH"\julia\"$Env:JULIA_VER"\bin\julia --inline=yes --check-bounds=yes --color=yes --project=@. -e 'import Pkg; Pkg.test(; coverage = true)'" - - exit $LASTEXITCODE - -.global_env_win10: &global_env_win10 - tags: - - windows10 - <<: *global_env_win - -.global_env_mac: &global_env_mac - tags: - - mac - script: - - $ARTENOLIS_SOFT_PATH/julia/$JULIA_VER/Contents/Resources/julia/bin/julia --inline=yes --check-bounds=yes --color=yes --project=@. -e 'import Pkg; Pkg.test(; coverage = true)' - -.global_build_apptainer: &global_build_apptainer - image: - name: "quay.io/singularity/singularity:$APPTAINER_DOCKER_TAG" - # the image entrypoint is the singularity binary by default - entrypoint: ["/bin/sh", "-c"] - tags: - - privileged - -# -# TESTS -# -# The "basic" required test that gets triggered for the basic testing, runs in -# any available docker and current julia -# - -docker:julia1.8: - stage: test - image: $CI_REGISTRY/r3/docker/julia-custom - script: - - julia --check-bounds=yes --inline=yes --project=@. -e "import Pkg; Pkg.test(; coverage = true)" - after_script: - - julia --project=test/coverage test/coverage/coverage-summary.jl - <<: *global_trigger_pull_request - -# -# The required compatibility test to pass on branches&tags before the docs get -# built & deployed -# - -linux:julia1.8: - stage: test - tags: - - slave01 - <<: *global_trigger_full_tests - <<: *global_julia18 - <<: *global_env_linux - -# -# Additional platform&environment compatibility tests -# - -windows10:julia1.8: - stage: test - <<: *global_trigger_full_tests - <<: *global_julia18 - <<: *global_env_win10 - -mac:julia1.8: - stage: test - <<: *global_trigger_full_tests - <<: *global_julia18 - <<: *global_env_mac - -# -# CONTAINERS -# - -apptainer-test: - stage: assets - script: | - alias apptainer=singularity - apptainer build gigasom-test.sif gigasom.def - <<: *global_build_apptainer - <<: *global_trigger_test_containers - -apptainer-release: - stage: assets - script: - - | - # build the container - alias apptainer=singularity - apptainer build gigasom.sif gigasom.def - - | - # push to GHCR - alias apptainer=singularity - export SINGULARITY_DOCKER_USERNAME="$GITHUB_ACCESS_USERNAME" - export SINGULARITY_DOCKER_PASSWORD="$GITHUB_ACCESS_TOKEN" - apptainer push gigasom.sif "oras://ghcr.io/$APPTAINER_GHCR_TAG:latest" - apptainer push gigasom.sif "oras://ghcr.io/$APPTAINER_GHCR_TAG:$CI_COMMIT_TAG" - <<: *global_build_apptainer - <<: *global_trigger_release_containers - -docker-test: - stage: assets - script: - - docker build -t "$DOCKER_HUB_TAG:testing" . - <<: *global_dind - <<: *global_trigger_test_containers - -docker-release: - stage: assets - script: - - docker build -t "$DOCKER_HUB_TAG:latest" . - # alias and push to docker hub - - docker tag "$DOCKER_HUB_TAG:latest" "$DOCKER_HUB_TAG:$CI_COMMIT_TAG" - - echo "$DOCKER_IO_ACCESS_TOKEN" | docker login --username "$DOCKER_IO_USER" --password-stdin - - docker push "$DOCKER_HUB_TAG:latest" - - docker push "$DOCKER_HUB_TAG:$CI_COMMIT_TAG" - # make 2 extra aliases and push to GHCR - - docker tag "$DOCKER_HUB_TAG:latest" "$DOCKER_GHCR_TAG:latest" - - docker tag "$DOCKER_HUB_TAG:latest" "$DOCKER_GHCR_TAG:$CI_COMMIT_TAG" - - echo "$GITHUB_ACCESS_TOKEN" | docker login ghcr.io --username "$GITHUB_ACCESS_USERNAME" --password-stdin - - docker push "$DOCKER_GHCR_TAG:latest" - - docker push "$DOCKER_GHCR_TAG:$CI_COMMIT_TAG" - <<: *global_dind - <<: *global_trigger_release_containers diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 7b2ea243..00000000 --- a/.travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -language: julia - -branches: - only: - - develop - - master - -os: - - linux - -julia: - - 1.0 - - 1.5 - -notifications: - email: false diff --git a/Project.toml b/Project.toml index 184b6e98..8ba1cd30 100644 --- a/Project.toml +++ b/Project.toml @@ -1,15 +1,15 @@ name = "GigaSOM" uuid = "a03a9c34-069e-5582-a11c-5c984cab887c" -version = "0.7.1" +version = "1.0.0" [deps] CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" -DistributedArrays = "aaf54ef3-cdf8-58ed-94cc-d582ad619b94" DistributedData = "f6a0035f-c5ac-4ad0-b410-ad102ced35df" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" +DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" FCSFiles = "d76558cf-badf-52d4-a17e-381ab0b0d937" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" NearestNeighbors = "b8a86587-4115-5ab1-83bc-aa920d37bbce" @@ -18,21 +18,37 @@ Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3" [compat] +AlgebraOfGraphics = "0.11.7" +Aqua = "0.8" CSV = "^0.5.12, 0.6, 0.7, 0.8, 0.10" +CairoMakie = "0.15.6" +Clustering = "0.15.8" DataFrames = "0.20, 0.21, 0.22, 1.0" Distances = "^0.8.2, 0.9, 0.10" -DistributedArrays = "^0.6.4" +Distributed = "1" DistributedData = "0.1.4, 0.2" Distributions = "^0.21.1, 0.22, 0.23, 0.24, 0.25" +DocStringExtensions = "0.9.5" +Downloads = "1" FCSFiles = "^0.1.1, 0.2" FileIO = "^1.0.7" JSON = "0.21" +LinearAlgebra = "1" NearestNeighbors = "^0.4.3" +Random = "1" +SHA = "0.7" +Serialization = "1" StableRNGs = "1.0" +Test = "1" XLSX = "0.8" julia = "1" [extras] +AlgebraOfGraphics = "cbdf2221-f076-402e-a563-3d30da359d67" +Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +Clustering = "aaaa29a8-35af-508c-8bc3-b662a17a0fe5" +Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" @@ -40,4 +56,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" XLSX = "fdbf4ff8-1666-58a4-91e7-1b58723a45e0" [targets] -test = ["JSON", "LinearAlgebra", "SHA", "Test", "XLSX"] +test = ["AlgebraOfGraphics", "Aqua", "CairoMakie", "Clustering", "Downloads", "JSON", "LinearAlgebra", "SHA", "Test", "XLSX"] diff --git a/README.md b/README.md index 4f69a316..3b62f70b 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,31 @@ -![GigaSOM.jl](https://webdav-r3lab.uni.lu/public/GigaSOM/img/logo-GigaSOM.jl.png?maxAge=0) -# GigaSOM.jl
Huge-scale, high-performance flow cytometry clustering +
+ +
-GigaSOM is a Julia toolkit for clustering and visualisation of really large cytometry data. Most generally, it can load FCS files, perform transformation and cleaning operations in their contents, run FlowSOM-style clustering, and visualize and export the results. GigaSOM is distributed and parallel in nature, which makes processing huge datasets a breeze -- a hundred of millions of cells with a few dozen parameters can be clustered and visualized in a few minutes. +# GigaSOM.jl: Huge-scale, high-performance flow cytometry clustering -| **Documentation** | **Test Coverage** | **CI** | **SciCrunch** | +[docs-img-stable]: https://img.shields.io/badge/docs-stable-blue.svg +[docs-url-stable]: https://lcsb-biocore.github.io/GigaSOM.jl +[docs-img-dev]: https://img.shields.io/badge/docs-latest-0af.svg +[docs-url-dev]: https://lcsb-biocore.github.io/GigaSOM.jl/dev/ +[docs-img-tutorials]: https://img.shields.io/badge/docs-tutorial-0dd.svg +[docs-url-tutorials]: https://lcsb-biocore.github.io/GigaSOM.jl/stable/tutorials/ + +[ci-img]: https://github.com/LCSB-BioCore/GigaSOM.jl/actions/workflows/ci.yml/badge.svg?branch=master +[ci-url]: https://github.com/LCSB-BioCore/GigaSOM.jl/actions/workflows/ci.yml + +[cov-img]: https://codecov.io/gh/LCSB-BioCore/GigaSOM.jl/branch/master/graph/badge.svg?token=H3WSWOBD7L +[cov-url]: https://codecov.io/gh/LCSB-BioCore/GigaSOM.jl + +[rrid-img]: https://img.shields.io/badge/RRID-SCR__019020-72c02c +[rrid-url]: https://scicrunch.org/resolver/RRID:SCR_019020 + +| **Documentation** | **Test Coverage** | **CI** | **SciCrunch RRID** | |:-----------------:|:-----------------:|:-----------------------------------------------------:|:--------:| -| [![doc](https://img.shields.io/badge/doc-GigaSOM-blue)](http://git.io/GigaSOM.jl) | [![coverage status](http://codecov.io/github/LCSB-BioCore/GigaSOM.jl/coverage.svg?branch=master)](http://codecov.io/github/LCSB-BioCore/GigaSOM.jl?branch=master) | [![linux](https://github.com/LCSB-BioCore/GigaSOM.jl/workflows/CI/badge.svg?branch=master)](https://github.com/LCSB-BioCore/GigaSOM.jl/actions) | [![rrid](https://img.shields.io/badge/RRID-SCR__019020-72c02c)](https://scicrunch.org/resolver/RRID:SCR_019020) | +| [![stable documentation][docs-img-stable]][docs-url-stable] [![dev documentation][docs-img-dev]][docs-url-dev] [![tutorials][docs-img-tutorials]][docs-url-tutorials] | [![coverage status][cov-img]][cov-url] | [![CI status][ci-img]][ci-url] | [![rrid][rrid-img]][rrid-url] | + +GigaSOM is a Julia toolkit for clustering and visualisation of really large datasets from flow cytometry. In overview, it can load FCS files, perform transformation and cleaning operations in their contents, run FlowSOM-style clustering, and visualize and export the results. GigaSOM is distributed and parallel in nature, which makes processing of huge datasets a breeze -- a few hundred of millions of cells with a few dozen parameters may be clustered and visualized in minutes. If you use GigaSOM.jl and want to refer to it in your work, use the following citation format (also available as BibTeX in [gigasom.bib](gigasom.bib)): @@ -14,37 +33,23 @@ If you use GigaSOM.jl and want to refer to it in your work, use the following ci # How to get started -## Prerequisites and requirements - -- **Operating system**: Use Linux (Debian, Ubuntu or centOS), MacOS, or Windows 10 as your operating system. GigaSOM has been tested on these systems. -- **Julia language**: In order to use GigaSOM, you need to install Julia 1.0 or higher. You can find the download and installation instructions for Julia [here](https://julialang.org/downloads/). -- **Hardware requirements**: GigaSOM runs on any hardware that can run Julia, and can easily use resources from multiple computers interconnected by network. For processing large datasets, you require to ensure that the total amount of available RAM on all involved computers is larger than the data size. - -:bulb: If you are new to Julia, it is adviseable to [familiarize youself with -the environment -first](https://docs.julialang.org/en/v1/manual/getting-started/). Use the full -Julia [documentation](https://docs.julialang.org) to solve various possible -language-related problems, and the [Julia package manager -docs](https://julialang.github.io/Pkg.jl/v1/getting-started/) to solve -installation-related difficulties. +As a major prerequisite, you will need to be able to run [Julia](https://julialang.org/). The environment is quite similar to other languages (R, Python, Matlab) and it is usually quite easy to start by following the [official user guide](). ## Installation -Using the Julia package manager to install GigaSOM is easy -- after starting Julia, type: +To install Julia, you can [download it from the official website](https://julialang.org/downloads/) or use your system's package manager to install one. Aim for using Julia version at least 1.6. +To install GigaSOM.jl, start Julia and type: ```julia import Pkg; Pkg.add("GigaSOM"); ``` -> All these commands should be run from Julia at the `julia>` prompt. - -Then you can load the GigaSOM package and start using it: - +After the installation finishes, you should be able to load GigaSOM package using: ```julia using GigaSOM ``` -The first loading of the GigaSOM package may take several minutes to complete due to precompilation of the sources, especially on a fresh Julia install. +The first loading of the GigaSOM package may take some time (a few minutes) to complete due to precompilation of the sources, especially if your Julia installation is new. ### Test the installation @@ -62,21 +67,21 @@ global_logger(ConsoleLogger(stderr, Logging.Debug)) ## How to use GigaSOM -A comprehensive documentation is [available online](https://lcsb-biocore.github.io/GigaSOM.jl/); several [introductory tutorials](https://lcsb-biocore.github.io/GigaSOM.jl/latest/tutorials/basicUsage/) of increasing complexity are also included. +A comprehensive documentation is [available online](https://lcsb-biocore.github.io/GigaSOM.jl/); several [introductory tutorials](https://lcsb-biocore.github.io/GigaSOM.jl/latest/tutorials/) of increasing complexity are also included. A very basic dataset (Levine13 from [FR-FCM-ZZPH](https://flowrepository.org/id/FR-FCM-ZZPH)) can be loaded, clustered and visualized as such: ```julia -using GigaSOM +import GigaSOM -params, fcsmatrix = loadFCS("Levine_13dim.fcs") # load the FCS file +params, fcsmatrix = GigaSOM.load_fcs("Levine_13dim.fcs") # load the FCS file exprs = fcsmatrix[:,1:13] # extract only the data columns with expression values -som = initGigaSOM(exprs, 20, 20) # random initialization of the SOM codebook -som = trainGigaSOM(som, exprs) # SOM training -clusters = mapToGigaSOM(som, exprs) # extraction of per-cell cluster IDs -e = embedGigaSOM(som, exprs) # EmbedSOM projection to 2D +som = GigaSOM.init(exprs, 20, 20) # initiali ze the SOM codebook randomly +som = GigaSOM.train(som, exprs) # train the SOM over the data +clusters = GigaSOM.assign(som, exprs) # extract per-cell cluster IDs +e = GigaSOM.embed(som, exprs) # make a 2D projection with EmbedSOM ``` The example loads the data, runs the SOM training (as in FlowSOM) and computes a 2D projection of the dataset (using EmbedSOM); the total computation time (excluding the possible precompilation of the libraries) should be around 15 seconds. @@ -95,10 +100,6 @@ savePNG("Levine13-CD4.png", expressionPalette(100, alpha=0.5))))) # colors for plotting (based on RdYlBu) ``` -The output may look like this (blue is negative expresison, red is positive): +The output may look like this (blue is negative expresison of CD4, red is positive): ![Levine13 embedding with CD4 highlighted](docs/src/assets/Levine13-CD4.png "Levine13/CD4") - -## Feedback, issues, questions - -Please follow the [contributing guide](.github/CONTRIBUTING.md) when you have questions, want to raise issues, or just want to leave us some feedback! diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index cbfea863..00000000 --- a/codecov.yml +++ /dev/null @@ -1,6 +0,0 @@ -coverage: - status: - project: - default: - threshold: 15 - patch: off diff --git a/docs/Project.toml b/docs/Project.toml index 2742e84b..10140e72 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,3 +1,13 @@ [deps] +AlgebraOfGraphics = "cbdf2221-f076-402e-a563-3d30da359d67" +CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +Clustering = "aaaa29a8-35af-508c-8bc3-b662a17a0fe5" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" +DistributedData = "f6a0035f-c5ac-4ad0-b410-ad102ced35df" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" DocumenterTools = "35a29f4d-8980-5a13-9543-d66fff28ecb8" +Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +GigaSOM = "a03a9c34-069e-5582-a11c-5c984cab887c" +Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" +XLSX = "fdbf4ff8-1666-58a4-91e7-1b58723a45e0" diff --git a/docs/make.jl b/docs/make.jl index 3949c708..ae9e7ae1 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,31 +1,48 @@ -using Documenter, GigaSOM +using Documenter, Literate, GigaSOM -makedocs(modules = [GigaSOM], - clean = false, - format = Documenter.HTML(prettyurls = !("local" in ARGS), - canonical = "https://lcsb-biocore.github.io/GigaSOM.jl/stable/", - assets = ["assets/gigasomlogotransp.ico"]), +tutorial_path = joinpath(@__DIR__, "src", "tutorials") + +tutorials = sort(filter(x -> endswith(x, ".jl"), readdir(tutorial_path, join = true))) + +for tutorial in tutorials + Literate.markdown( + tutorial, + tutorial_path, + repo_root_url = "https://github.com/LCSB-BioCore/GitaSOM.jl/blob/master", + ) +end + +tutorial_mds = "tutorials/" .* first.(splitext.(basename.(tutorials))) .* ".md" + +withenv("COLUMNS" => 150) do + makedocs( + modules = [GigaSOM], + format = Documenter.HTML( + ansicolor = true, + canonical = "https://lcsb-biocore.github.io/GigaSOM.jl/stable/", + assets = ["assets/gigasomlogotransp.ico"], + example_size_threshold = nothing, + size_threshold_warn = 10 * 1024 * 1024, + size_threshold = nothing, + ), sitename = "GigaSOM.jl", - authors = "The developers of GigaSOM.jl", - linkcheck = !("skiplinks" in ARGS), pages = [ - "Home" => "index.md", - "Background" => "background.md", - "Tutorial" => [ - "Introduction" => "tutorials/basicUsage.md", - "Cytometry data" => "tutorials/processingFCSData.md", - "Advanced distributed processing" => "tutorials/distributedProcessing.md", - "Conclusion" => "tutorials/whereToGoNext.md", - ], - "Functions" => "functions.md", - "How to contribute" => "howToContribute.md", - ], - ) + "Home" => "index.md", + "Background" => "background.md", + "Tutorials" => [ + "Contents" => "tutorials.md" + tutorial_mds + ], + "Further reading" => "nextsteps.md", + "Reference" => "reference.md", + ], + warnonly = [:linkcheck, :cross_references, :missing_docs], #TODO fix + ) +end deploydocs( repo = "github.com/LCSB-BioCore/GigaSOM.jl.git", target = "build", branch = "gh-pages", - devbranch = "develop", - push_preview = true + devbranch = "master", ) diff --git a/docs/src/assets/parallel.png b/docs/src/assets/parallel.png deleted file mode 100644 index c6f8a06a..00000000 Binary files a/docs/src/assets/parallel.png and /dev/null differ diff --git a/docs/src/background.md b/docs/src/background.md index 4c737085..a169a439 100644 --- a/docs/src/background.md +++ b/docs/src/background.md @@ -2,99 +2,87 @@ ## Introduction -Flow cytometry clustering for several hundred million cells has long been hampered -by software limitations. Julia allows us to go beyond these limits. -Through the high-performance GigaSOM.jl package, we gear up for huge-scale -flow cytometry analysis, softening these limitations with an innovative approach -on the existing algorithm, by implementing parallel computing with stable and +Flow cytometry clustering for several hundred million cells has long been +hampered by software limitations. Julia allows us to go beyond these limits. +Through the high-performance GigaSOM.jl package, we gear up for huge-scale flow +cytometry analysis, softening these limitations with an innovative approach on +the existing algorithm, by implementing parallel computing with stable and robust mathematical models that would run with an HPC cluster, allowing us to run cytometry datasets as big as 500 million cells. ## Multidimensional Data Multidimensional data is a term used when combining four, five (or more) -characteristics simultaneously to create a data space where each measured cell parameter -becomes a dimension in the multidimensional space.\ -The data generated by Flow Cytometry, Mass Cytometry or RNA-seq are good examples of -multidimensional datasets, as they combine all information from all characteristics to -create a multidimensional data space that preserves the integrity of the relationships -between each of the parameters.[[1]](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6092027/) +characteristics simultaneously to create a data space where each measured cell +parameter becomes a dimension in the multidimensional space. + +The data generated by Flow Cytometry, Mass Cytometry or RNA-seq are good +examples of multidimensional datasets, as they combine all information from all +characteristics to create a multidimensional data space that preserves the +integrity of the relationships between each of the +parameters.[[1]](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6092027/) ### Flow and Mass Cytometry -The use of flow cytometry has grown substantially in the past decade, mainly due to -the development of smaller, user-friendly and less expensive instruments, -but also to the increase of clinical applications, like cell counting, cell sorting, -detection of biomarkers or protein engineering. -Flow cytometry is an immunophenotyping technique used to identify and quantify the -cells of the immune system by analysing their physical and chemical characteristics -in a fluid. These cells are stained with specific, fluorescently labelled antibodies -and then analysed with a flow cytometer, where the fluorescence intensity is measured -using lasers and photodetectors. [[2]](http://clinchem.aaccjnls.org/content/46/8/1221) -More recently, a variation of flow cytometry called mass cytometry (CyTOF) was introduced, -in which antibodies are labelled with heavy metal ion tags rather than fluorochromes, -breaking the limit of multiplexing capability of FACS (fluorescence-activated cell sorting) -and allowing the simultaneous quantification of 40+ protein parameters within each single cell. -The ability of flow cytometry and mass cytometry to analyse individual cells at high-throughput -scales makes them ideal for multi-parameter cell analysis and high-speed sorting. [[3]](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4860251/) +The use of flow cytometry has grown substantially in the past decade, mainly +due to the development of smaller, user-friendly and less expensive +instruments, but also to the increase of clinical applications, like cell +counting, cell sorting, detection of biomarkers or protein engineering. + +Flow cytometry is an immunophenotyping technique used to identify and quantify +the cells of the immune system by analysing their physical and chemical +characteristics in a fluid. These cells are stained with specific, +fluorescently labelled antibodies and then analysed with a flow cytometer, +where the fluorescence intensity is measured using lasers and photodetectors. +[[2]](http://clinchem.aaccjnls.org/content/46/8/1221) + +More recently, a variation of flow cytometry called mass cytometry (CyTOF) was +introduced, in which antibodies are labelled with heavy metal ion tags rather +than fluorochromes, breaking the limit of multiplexing capability of FACS +(fluorescence-activated cell sorting) and allowing the simultaneous +quantification of 40+ protein parameters within each single cell. The ability +of flow cytometry and mass cytometry to analyse individual cells at +high-throughput scales makes them ideal for multi-parameter cell analysis and +high-speed sorting. +[[3]](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4860251/) ![MassCytometry](assets/masscytometry.png)\ -*General workflow of cell analysis in a mass cytometer. The cells are introduced into the nebulizer via a narrow capillary. As the cells exit from the nebulizer they are converted to a fine spray of droplets, which are then carried into the plasma where they are completely atomized and ionized. The resulting ion cloud is filtered and selected for positive ions of mass range 80–200 and measured in a TOF chamber. The data are then converted to .fcs format and analyzed using traditional flow cytometry software.* [[4]](http://dmd.aspetjournals.org/content/43/2/227) +*General workflow of cell analysis in a mass cytometer. The cells are +introduced into the nebulizer via a narrow capillary. As the cells exit from +the nebulizer they are converted to a fine spray of droplets, which are then +carried into the plasma where they are completely atomized and ionized. The +resulting ion cloud is filtered and selected for positive ions of mass range +80–200 and measured in a TOF chamber. The data are then converted to .fcs +format and analyzed using traditional flow cytometry software.* +[[4]](http://dmd.aspetjournals.org/content/43/2/227) ## Self-organising maps (SOMs) Self-organising maps (also referred to as SOMs or *Kohonen* maps) are -artificial neural networks introduced by Teuvo Kohonen in the 1980s. -Despite of their age, SOMs are still widely used as an easy and robust -unsupervised learning technique for analysis and visualisation of high-dimensional data. -The SOM algorithm maps high-dimensional vectors into a lower-dimensional grid. -Most often the target grid is two-dimensional, resulting into intuitively interpretable maps. -After initializing a SOM grid of size n*n, each node is initialized with a random sample (row) -from the dataset (training data). For each input vector (row) in the training data the distance -to each node in the grid is calculated, using Chebyshev distance or Euclidean distance equations, -where the closest node is called BMU (best matching unit). The row is subsequently assigned to the -BMU making it move closer to the input data, influenced by the learning rate and neighborhood Gaussian -function, whilst the neighborhood nodes are also adjusted closer to the BMU. This training step is -repeated for each row in the complete dataset. After each iteration (epoch) the radius of the -neighborhood function is reduced. After n epochs, clusters of nodes should have formed and as a -final step, consensus cluster is used to reduce the data (SOM nodes) into m clusters. [[5]](https://ieeexplore.ieee.org/document/58325) +artificial neural networks introduced by Teuvo Kohonen in the 1980s. Despite +of their age, SOMs are still widely used as an easy and robust unsupervised +learning technique for analysis and visualisation of high-dimensional data. -![SOMs](assets/soms.png)\ -*Example of a self-organizing network with five cluster units, Yi, and seven input units, Xi. The five cluster units are arranged in a linear array.* [[6]](http://mnemstudio.org/neural-networks-kohonen-self-organizing-maps.htm) +The SOM algorithm maps high-dimensional vectors into a lower-dimensional grid. +Most often the target grid is two-dimensional, resulting into intuitively +interpretable maps. After initializing a SOM grid of size `n*n`, each node is +initialized with a random sample (row) from the dataset (training data). For +each input vector (row) in the training data the distance to each node in the +grid is calculated, using Chebyshev distance or Euclidean distance equations, +where the closest node is called BMU (best matching unit). -## Implementation +The row is subsequently assigned to the BMU, making it move closer to the input +data, influenced by the learning rate and neighborhood Gaussian function, +whilst the neighborhood nodes are also adjusted closer to the BMU. This +training step is repeated for each row in the complete dataset. After each +iteration (epoch) the radius of the neighborhood function is reduced. After n +epochs, clusters of nodes should have formed and as a final step, consensus +cluster is used to reduce the data (SOM nodes) into m clusters. +[[5]](https://ieeexplore.ieee.org/document/58325) -The high-performance [GigaSOM.jl](https://github.com/LCSB-BioCore/GigaSOM.jl) package -enables the analysis and clustering of huge-scale flow cytometry data because it is HPC-ready -and written in Julia, prepared to handle very large datasets. -Also, the GigaSOM.jl package, provides training and visualization functions for -Kohonen's self-organizing maps for Julia. -Training functions are implemented in pure Julia, without depending on additional binary libraries. -The SOM algorithm maps high-dimensional vectors into a lower-dimensional grid. -Most often, the target grid is two-dimensional, resulting into intuitively interpretable maps. -The general idea is to receive huge-scale .fcs data files as an input, load and -transform them accordingly to enable the analysis and clustering of these data -and automatically determine the required number of cell populations, -and their sensitivity and specificity using parallel computing. -In order to achieve this, GigaSOM.jl implementes a batch SOM algorithm, in which -the weight vectors are only updated at the end of each epoch, and the BMU is calculated -using the weight vectors from the previous epoch. Since the weight updates are not recursive, -the order of the inputs does not affect the final result. In addition, there is no -learning rate coefficient, which reduces the data dependency. This means that, -compared to the original one, the batch algorithm has faster convergence, requires less computing, -and it is the only one suitable for parallel computing. -Parallel computing is the last step of our package implementation and two possible -approaches exist for this kind of computation: the Model Parallelism and the Data Parallelism. -In the Model Parallelism approach, the algorithm sends the same data to all the processes, -and then each process is responsible for estimating different parameters and exchange their -estimates with each other to come up with the right estimate for all the parameters. -In the Data Parallelism approach, the algorithm distributes the data between different processes, -and then each process independently tries to estimate the same parameters and then -exchange their estimates with each other to come up with the right estimate. -On this project, we use the Data Parallelism approaches because our nodes grid -is too small for the Model Parallelism approach. - -![parallel](assets/parallel.png)\ -*Data Parallelism vs Model Parallelism* [[7]](https://www.slideshare.net/JunyoungPark22/common-design-for-distributed-machine-learning) +![SOMs](assets/soms.png)\ +*Example of a self-organizing network with five cluster units, Yi, and seven +input units, Xi. The five cluster units are arranged in a linear array.* +[[6]](http://mnemstudio.org/neural-networks-kohonen-self-organizing-maps.htm) diff --git a/docs/src/howToContribute.md b/docs/src/howToContribute.md deleted file mode 100644 index 3ee843e0..00000000 --- a/docs/src/howToContribute.md +++ /dev/null @@ -1,125 +0,0 @@ -# How to contribute to/develop GigaSOM - -If you want to contribute to the `GigaSOM` package, please fork the present -repository by following [these instructions](https://help.github.com/en/github/getting-started-with-github/fork-a-repo). - -## Step 1: Retrieve a local version of GigaSOM - -There are two ways that you can retrieve a local copy of the package: one is to -manually clone the forked repository, and the second one is to use the -intergrated Julia package manager. - -### Option 1: Manually clone your fork - -:warning: Please make sure you have _forked_ the repository, as described above. - -You can do this as follows from the command line: - -```bash -$ git clone git@github.com:yourUsername/GigaSOM.jl.git GigaSOM.jl -$ cd GigaSOM.jl -$ git checkout -b yourNewBranch origin/develop -``` - -where `yourUsername` is your Github username and `yourNewBranch` is the name of a new branch. - -Then, in order to develop the package, you can install your cloned version as -follows (make sure you are in the `GigaSOM.jl` directory): - -```julia -(v1.1) pkg> add . -``` - -(press `]` to get into the packaging environment) - -This adds the `GigaSOM.jl` package and all its dependencies. You can verify -that the installation worked by typing: - -```julia -(v1.1) pkg> status -``` - -If everything went smoothly, this should print something similar to: - -```julia -(v1.1) pkg> status - Status `~/.julia/environments/v1.1/Project.toml` - [a03a9c34] GigaSOM v0.0.5 #yourNewBranch (.) -``` - -Now, you can readily start using the `GigaSOM` module: - -```julia -julia> using GigaSOM -``` - -### Option 2: Use the Julia package manager - -When you are used to using the Julia package manager for developing or -contributing to packages, you can type: - -```julia -(v1.1) pkg> dev GigaSOM -``` - -This will install the `GigaSOM` package locally and check it out for -development. You can check the location of the package with: - -```julia -(v1.1) pkg> status - Status `~/.julia/environments/v1.1/Project.toml` - [a03a9c34] GigaSOM v0.0.5 [`~/.julia/dev/GigaSOM`] -``` - -The default location of the package is `~/.julia/dev/GigaSOM`. - -You can then set your remote by executing these commands in a regular shell: - -```bash -$ cd ~/.julia/dev/GigaSOM -$ git remote rename origin upstream # renames the origin as upstream -$ git remote add origin git@github.com:yourUsername/GigaSOM.jl.git -$ git fetch origin -``` - -where `yourUsername` is your Github username. - -:warning: Make sure that your fork exists under `github.com/yourUsername/GigaSOM.jl`. - -Then, checkout a branch `yourNewBranch`: - -```bash -$ cd ~/.julia/dev/GigaSOM -$ git checkout -b yourNewBranch origin/develop -``` - -Then, you can readily use the `GigaSOM` package: - -```julia -julia> using GigaSOM -``` - -After making changes, precompile the package: - -```julia -(v1.1) pkg> precompile -``` - -## Step 2: Activate GigaSOM - -:warning: Please note that you cannot use the dependencies of GigaSOM directly, -unless they are installed separately or the environment has been activated: - -```julia -(v1.1) pkg> activate . -(GigaSOM) pkg> instantiate -``` - -Now, the environment is activated (you can see it with the prompt change -`(GigaSOM) pkg>`). Now, you can use the dependency. For instance: - -```julia -julia> using DataFrames -``` - -:warning: If you do not `activate` the environment before using any of the dependencies, you will see a red error messages prompting you to install the dependency explicity. diff --git a/docs/src/index.md b/docs/src/index.md index f8d4390c..9d06c684 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,67 +1,7 @@ -![GigaSOM.jl](https://webdav-r3lab.uni.lu/public/GigaSOM/img/logo-GigaSOM.jl.png?maxAge=0) -# GigaSOM.jl - Huge-scale, high-performance flow cytometry clustering in Julia +# GigaSOM.jl documentation - -GigaSOM.jl allows painless analysis of huge-scale clinical studies, scaling down the software limitations that usually prevent work with large datasets. It can be viewed as a work-alike of FlowSOM, suitable for loading billions of cells and running the analyses in parallel on distributed computer clusters, to gain speed. Most importantly, GigaSOM.jl scales horizontally -- data volume limitations and memory limitations can be solved just by adding more computers to the cluster. That makes it extremely easy to exploit HPC environments, which are becoming increasingly common in computational biology. - -```@raw html - -
- -
- Evolution of the GigaSOM.jl repository (2019-2020) -
-``` - -### Features - -- Horizontal scalability to literal giga-scale datasets (``10^9`` cells!) -- HPC-ready, support for e.g. Slurm -- Standard support for distributed loading, scaling and transforming the FCS3 files -- Batch-SOM based GigaSOM algorithm for clustering -- EmbedSOM for visualizations - -### Background - -You can learn more about the background of GigaSOM.jl in these sections: - -```@contents -Pages = ["background.md"] +```@autodocs +Modules = [GigaSOM] +Pages = ["src/GigaSOM.jl"] ``` - -### How to get started? - -You can follow our extensive tutorials here: - -```@contents -Pages = ["tutorials/basicUsage.md", - "tutorials/processingFCSData.md", - "tutorials/distributedProcessing.md", - "tutorials/whereToGoNext.md" -] -``` - -### Functions - -A full reference to all functions is given here: - -```@contents -Pages = ["functions.md"] -``` - -### How to contribute? - -If you want to contribute, please read these guidelines first: - -```@contents -Pages = ["howToContribute.md"] -``` \ No newline at end of file diff --git a/docs/src/tutorials/whereToGoNext.md b/docs/src/nextsteps.md similarity index 75% rename from docs/src/tutorials/whereToGoNext.md rename to docs/src/nextsteps.md index 11e8fdda..d2caeb20 100644 --- a/docs/src/tutorials/whereToGoNext.md +++ b/docs/src/nextsteps.md @@ -6,18 +6,18 @@ GigaSOM.jl works internally. With real data, you may encounter situations that require deeper digging in parameters and GigaSOM internals. We list several of the most frequently used ones: -- You may want to increase the size of SOM in [`initGigaSOM`](@ref) to get more +- You may want to increase the size of SOM in [`init`](@ref) to get more precise clusters. - You may speed up the computation a lot by using neighborhood-indexing structures -- see package `NearestNeighbors` and parameters `knnTreeFun` of - functions [`trainGigaSOM`](@ref) and [`embedGigaSOM`](@ref) + functions [`train`](@ref) and [`embed`](@ref) - It is adviced to try different settings of SOM training -- of the arguments - of [`trainGigaSOM`](@ref), try modifying the starting/finishing radius + of [`train`](@ref), try modifying the starting/finishing radius (`rStart`, `rFinal`), using a different radius decay (parameter `radiusFun`, try e.g. `linearRadius`) or try a different neighborhood (`kernelFun` and `somDistFun`) or a completely different metric. - You can get a sharper or smoother embedding by varying the amount of neighbors (`k`) and smoothing of the neighborhood (`smooth`) of - [`embedGigaSOM`](@ref). + [`embed`](@ref). - For plotting of really huge data, you may want to try - [GigaScatter](https://github.com/LCSB-BioCore/GigaScatter.jl). + [GigaScatter.jl](https://github.com/LCSB-BioCore/GigaScatter.jl). diff --git a/docs/src/functions.md b/docs/src/reference.md similarity index 68% rename from docs/src/functions.md rename to docs/src/reference.md index 95e64b05..079b7abe 100644 --- a/docs/src/functions.md +++ b/docs/src/reference.md @@ -11,19 +11,19 @@ Pages = ["structs.jl"] ```@autodocs Modules = [GigaSOM] -Pages = ["input.jl", "process.jl", "splitting.jl", "dataops.jl"] +Pages = ["load.jl", "data_utils.jl", "split.jl"] ``` ## SOM training ```@autodocs Modules = [GigaSOM] -Pages = ["core.jl", "trainutils.jl"] +Pages = ["som.jl", "som_utils.jl"] ``` ## Embedding ```@autodocs Modules = [GigaSOM] -Pages = ["embedding.jl"] +Pages = ["embed.jl"] ``` diff --git a/docs/src/tutorials.md b/docs/src/tutorials.md new file mode 100644 index 00000000..3341d86d --- /dev/null +++ b/docs/src/tutorials.md @@ -0,0 +1,7 @@ + +# Tutorials + +```@contents +Pages = filter(x -> endswith(x, ".md"), readdir("tutorials", join=true)) +Depth = 2 +``` diff --git a/docs/src/tutorials/01-basic.jl b/docs/src/tutorials/01-basic.jl new file mode 100644 index 00000000..3c05587c --- /dev/null +++ b/docs/src/tutorials/01-basic.jl @@ -0,0 +1,125 @@ + +# # Starting up +# +# For the installation of Julia or GigaSOM.jl please refer to the installation +# instructions in the +# [README](https://github.com/LCSB-BioCore/GigaSOM.jl?tab=readme-ov-file#installation). +# +# ## High-level overview +# GigaSOM provides functions that allow straightforward loading of FCS files into +# matrices, preparing these matrices for analysis, and running SOM and related +# function on the data. +# +# The main functions, listed by category, include: +# +# - [`load_fcs`](@ref) and [`load_fcs_distributed`](@ref) for loading the data +# - [`fcs_column_metadata`](@ref) and [`fcs_metadata_marker_names`](@ref) for +# accessing the information about data columns stored in FCS files +# - `dselect`, `dtransform_asinh`, `dscale` and similar functions from +# [`DistributedData.jl`](https://lcsb-biocore.github.io/DistributedData.jl/stable/) +# for transforming, scaling and preparing the data +# - [`init`](@ref) for initializing the self-organizing map, [`train`](@ref) +# for running the SOM training, [`assign`](@ref) for classification using the +# trained SOM, and [`embed`](@ref) for dimensionality reduction to 2D +# +# Parallel processing is perfurmed using the `Distributed` package -- if you +# add more "workers" using the `addprocs` function, GigaSOM will automatically +# react to that situation, split the data among the processes and run parallel +# versions of all algorithms. +# +# ### Horizontal scaling +# +# While all functions work well on simple data matrices, the main aim of +# GigaSOM is to let the users enjoy the cluster-computing resources. All +# functions also work on data that are "scattered" among workers (i.e. each +# worker only holds a portion of the data loaded in the memory). This dataset +# description is stored in [`Dinfo`](@ref) structure. Most of the functions +# above in fact accept the `Dinfo` as argument, and often return another +# `Dinfo` that describes the scattered result. +# +# Most importantly, using `Dinfo` *prevents memory exhaustion at the +# master node*, which is a critical feature required to handle huge datasets. +# +# You can always collect the scattered data back into a matrix (if it fits to +# your RAM) with `gather_array`, and utilize many other functions to manipulate +# it, including e.g. `dmapreduce` for easily running parallel computations, or +# `dstore` for saving and restoring the dataset paralelly. +# +# ## Minimal working example +# +# First, load GigaSOM: + +using GigaSOM + +# For the purpose of demonstration, we will create a small random dataset. This +# code generates a 4D hypercube of "size" 10 with gaussian clusters at the +# cube's vertices: + +import Random #src +Random.seed!(12345) #src +d = randn(10000, 4) .+ rand(0:1, 10000, 4) .* 10; + +# The SOM (of size 20×20) is created and trained as such: + +som = init(d, 20, 20) +som = train(som, d) + +# (Note that SOM initialization is randomized; if you want to get the same +# results everytime, set the random seed before initialization: + +som = init(d, 20, 20, seed = 12345) +som = train(som, d) + +# You can now see the SOM codebook (your numbers will likely be different): + +som.codes + +@test size(som.grid) == (400, 2) #src +@test size(som.codes) == (400, 4) #src +@test minimum(som.codes) > -3 #src +@test maximum(som.codes) < 13 #src + +# This information can be used to categorize the dataset into clusters: + +assign(som, d) + +@test length(assign(som, d)) == 10000 #src + +# In the result, `index` is a cluster ID for the original datapoint from `d` at +# the same row. (As in the previous case, your numbers may differ.) +# +# Finally, you can use EmbedSOM dimensionality reduction to convert all +# multidimensional points to 2D; which can eventually be used to create a +# good-looking 2D scatterplot. +e = embed(som, d) + +@test minimum(e) > -3 #src +@test maximum(e) < 22 #src + +# The 2D coordinates may be plotted using any standard plotting library. In the +# following example we show how to do that with AlgebraOfGraphics package: + +using AlgebraOfGraphics, CairoMakie, DataFrames + +embedded_data = [DataFrame(e, [:x, :y]) DataFrame(d, [:v1, :v2, :v3, :v4])] + +draw(data(embedded_data)*mapping(:x, :y, color = :v1)*visual(Scatter, markersize = 2)) + +# The output shows all 16 ($16 = 2^d$ where $d =4$), colored by their position +# in the 1st dimension in the original space. +# +# For complete clarity on the underlying data structures, we can also plot the +# self-organizing map that "describes" the space: + +som_data = [DataFrame(som.grid, [:x, :y]) DataFrame(som.codes, [:v1, :v2, :v3, :v4])] + +draw(data(som_data)*mapping(:x, :y, color = :v1)*visual(Scatter, markersize = 20)) + +# To be completely clear about cluster occupancy, we can also compute the +# assignment counts and highlight them in the plot: +som_data[!, :count] .= 0 +for i in assign(som, d) + som_data[i, :count] += 1 +end + +draw(data(som_data)*mapping(:x, :y, color = :v1, markersize = :count)*visual(Scatter)) diff --git a/docs/src/tutorials/02-flow-data.jl b/docs/src/tutorials/02-flow-data.jl new file mode 100644 index 00000000..09ab02aa --- /dev/null +++ b/docs/src/tutorials/02-flow-data.jl @@ -0,0 +1,241 @@ + +# # Tutorial 2: Working with cytometry data +# +# You can load any FCS file using [`load_fcs`](@ref) function. For example, the +# Levine dataset ([obtainable here](https://flowrepository.org/id/FR-FCM-ZZPH)) +# may be loaded as such: + +using GigaSOM +import Downloads: download + +isfile("Levine_13dim.fcs") || download( + "https://github.com/lmweber/benchmark-data-Levine-13-dim/raw/refs/heads/master/data/Levine_13dim.fcs", + "Levine_13dim.fcs", +) + +params, expressions = load_fcs("Levine_13dim.fcs") + +@test length(params)==111 #src +@test size(expressions)==(167044, 14) #src + +# `params` will now contain the list of FCS parameters; you can parse a lot of +# interesting information from it using the [`fcs_column_metadata`](@ref) +# function: + +fcs_column_metadata(params) + +@test fcs_column_metadata(params).N[14] == "label" #src + +# `expressions` is a matrix with cell expressions, one cell per row, one marker per +# column. If you want to run SOM analysis on it, you can cluster and visualize +# it just as in the previous tutorial, with one exception- we start with +# cutting off the `label` column that contains `NaN` values: + +expressions = expressions[:, 1:13] +som = init(expressions, 16, 16, seed = 12345) +som = train(som, expressions) +clusters = assign(som, expressions) +e = embed(som, expressions) + +# ... etc, you can save the results and plot them as needed. We demonstrate the +# plotting below. + +# +# ## Working with distributed data +# +# Usual experiments produce multiple FCS files, and distributed or parallel +# processing is very helpful in crunching through all the data. +# +# To load multiple FCS files, use [`load_fcs_distributed`](@ref). This function +# works well in the "usual" single-process environment, but additionally it is +# designed to handle situations when the data is too big to fit into memory, +# and attempts to split them into available distributed workers workers. +# +# For the purpose of data distribution, you need to identify each dataset by an +# unique **dataset name** that will be used for identifying your loaded data in +# the cluster environment. The dataset name is a simple Julia symbols; +# basically a variable name that is prefixed with a `:` colon. +# +# For example, we can load the Levine13 dataset as such: + +datainfo = load_fcs_distributed(:levine, ["Levine_13dim.fcs"]) + +# Expectably, if you have more files, just write their names into the array and +# the function will handle the rest. +# +# The result `datainfo` carries informaton about your selected dataset name and +# its distribution among the cluster. It can be used just as the "data" +# parameter in all SOM-related functions again; e.g. as `train(som, datainfo)`. +# +# The following example exploits the possibility to actually split the data, +# and processes the Levine dataset parallelly on 2 workers: + +using Distributed, DistributedData +addprocs(2) # add any number of CPUs/tasks/workers you have available +@everywhere using GigaSOM # load GigaSOM also on the workers + +datainfo = load_fcs_distributed(:levine, ["Levine_13dim.fcs"]) + +# We select only columns that contain expressions (in the dataset, column 14 +# contains labels): +dselect(datainfo, Vector(1:13)) + +# Training may continue as usual: +som = init(datainfo, 20, 20) +som = train(som, datainfo) + +# To prevent memory overload of the "master" computation node, the results of +# all per-cell operations are also stored in distributed "data info" objects. +# In this case, the following code does the embedding, but leaves the resulting +# data safely scattered among the cluster: +e = embed(som, datainfo) + +# If you are sure you have enough RAM, you can collect the data to the master +# node. (In case of the relatively small Levine13 dataset, you very probably +# have the required 2.5MB of RAM, but there are many larger datasets.) +e = gather_array(e) + +# ## Working with larger datasets +# +# In this example we will use a subset of the Cytometry data [by Bodenmiller et +# al.](https://doi.org/10.1038/nbt.2317). This dataset contains samples from +# peripheral blood mononuclear cells (PBMCs) in unstimulated and stimulated +# conditions for 8 healthy donors. +# +# 10 cell surface markers (lineage markers) are used to identify different cell +# populations. The dataset is described in two files: +# +# - `PBMC8_panel.xlsx` (with antigen names categorized as lineage markers and +# functional markers) +# - `PBMC8_metadata.xlsx` (file names, sample IDs, condition IDs and patient +# IDs) + +### Download and prepare the dataset + +# The example data can be downloaded from +# [imlspenticton.uzh.ch/robinson_lab/cytofWorkflow/](http://imlspenticton.uzh.ch/robinson_lab/cytofWorkflow/). For availability reasons, we mirror them also in GigaSOM repository. +# +# You can fetch the files directly from within Julia: + +dataFiles = ["PBMC8_metadata.xlsx", "PBMC8_panel.xlsx", "PBMC8_fcs_files.zip"] +for f in dataFiles + if !isfile(f) + download( + "https://github.com/LCSB-BioCore/GigaSOM.jl/raw/refs/heads/mirrored-data/"*f, + f, + ) + end +end + +run(`unzip PBMC8_fcs_files.zip`) + +# The metadata is present in external files; we read it into a `DataFrame` and +# extract information about FCS data columns from there. First, we read the +# actual content using the XLSX package: + +import XLSX +using DataFrames +md = DataFrame(XLSX.readtable("PBMC8_metadata.xlsx", "Sheet1", infer_eltypes = true)) +panel = DataFrame(XLSX.readtable("PBMC8_panel.xlsx", "Sheet1", infer_eltypes = true)) + +# After that, we can get the parameter structure from the first FCS files: +_, fcsParams = GigaSOM.read_fcs_header(md[1, :file_name]) + +# ...and continue with extracting marker names using the prepared functions: +_, fcsAntigens = GigaSOM.fcs_metadata_marker_names(fcs_column_metadata(fcsParams)) + +# Now, TO see which antigens we want to use (assume we want only the lineage +# markers): +antigens = panel[panel[:, :Lineage] .== 1, :Antigen] + +# Finally, it is often useful to make the names a bit more Julia-friendly and +# predictable: +clean_names!(antigens) +clean_names!(fcsAntigens) + +# ### Load and prepare the data +# +# Now we have the vector of `fcsAntigens` that the FCS files store, and list of +# `antigens` that we want to analyze. We continue by loading the data, reducing +# it to the desired antigens and transforming it a bit: +di = load_fcs_distributed(:pbmc8, md[:, :file_name]) + +# (If data distribution and parallelization is required, you must add parallel +# workers using `addprocs` **before** this step.) + +# Now that the data is loaded, let's prepare them a bit by reducing to actual +# interesting columns, transformation and scaling. First, select only the +# columns that correspond to the lineage antigens we have prepared before: +dselect(di, fcsAntigens, antigens) +cols = Vector(eachindex(antigens)) # shortcut for "all rows" + +# Perform `asinh` transformation on all data in the dataset, '5' here is the +# cofactor for the transformation: +dtransform_asinh(di, cols, 5) + +# Normalize all dataset columns to zero mean and unit variance: +dscale(di, cols) + +# ### Train a self-organizing map (SOM) +# +# With the data prepared, running the SOM algorithm is straightforward: +som = init(di, 20, 20, seed = 12345) +som = train(som, di, epochs = 20) + +# Finally, we can calculate the clustering: +som_clusters = assign(som, di) + +# Among other, the ability to collect the data also allows us to plot it using +# general plotting libraries. (With much larger datasets, a more complicated +# setup is needed where the plotting is done by parts.) In this case, we simply +# collect the expressions and embedding into a single dataframe and plot them +# as usual: + +using AlgebraOfGraphics, CairoMakie, DataFrames + +e = embed(som, di) + +plot_data = [DataFrame(gather_array(di), antigens) DataFrame(gather_array(e), [:x, :y])] + +draw( + data(plot_data)*mapping(:x, :y, color = :CD3)*visual(Scatter, markersize = 2), + scales(Color = (; colormap = :RdYlBu)), +) + + +# ### FlowSOM-style metaclustering +# +# One disadvantage of SOMs is that they output a large amount of small clusters +# that are relatively hard to interpret manually. FlowSOM improved that +# situation by running a "clustering on clusters" (metaclustering) that address +# the problem. +# +# In this example, we reduce the original 256 small clusters from 16x16 SOM to +# only 10 "metaclusters", using the standard hierarchical clustering: + +using Clustering +import Distances +metaclusters = cutree( + k = 10, + hclust(linkage = :average, GigaSOM.distance_matrix(Distances.Euclidean())(som.codes)), +) + +# The `metaClusters` represent membership of the SOM codes in cluster; these +# can be expanded to membership of all cells using [`assign`](@ref): + +clustering = gather_array(assign(som, di), free = true) +plot_data[!, :cluster] = Symbol.(metaclusters[clustering]) + +# The plot data column `cluster` now contains classification of each cell in +# the dataset. We can plot it as follows: +draw( + data(plot_data)*mapping(:x, :y, color = :cluster)*visual( + Scatter, + markersize = 2, + colormap = :Set3_10, + ), +) + +# (Note: The argument `free=true` of `gather_array` automatically removes the +# distributed data from workers after collecting, which frees the memory for +# use with other datasets.) diff --git a/docs/src/tutorials/03-distributed-data.jl b/docs/src/tutorials/03-distributed-data.jl new file mode 100644 index 00000000..0587008b --- /dev/null +++ b/docs/src/tutorials/03-distributed-data.jl @@ -0,0 +1,139 @@ + +# # Tutorial 3: Distributed data processing and statistics + +# If you can get the data on a single machine, computation of various +# statistics can be performed using standard Julia functions. With large +# datasets that do not fit on a computer, things get more complicated. Luckily, +# many statistics and algorithms possess parallel, map-reduce-style +# implementations that can be used to address this problem. +# +# Here, we show how to use DistributedData package to run various operations on +# the datasets scattered over many computation nodes. For demonstration, we use +# a dataset that contains the "hypercube of clusters": + +using DistributedData +using Distributed + +import Random #src +Random.seed!(12345) #src +d = randn(10000, 4) .+ rand(0:1, 10000, 4) .* 10; + +# First, we distribute the dataset over the available workers to obtain a +# "handle": + +di = scatter_array(:distributed_d, d, workers()) + +# There are many functions to work with such data by "handles". For example, +# the `dstat` function computes sample statistics without fetching all data to +# a single node. You can use it in a way similar to aforementioned `dselect` +# and `dtransform_asinh`. The following code extracts means and standard +# deviations from the first 3 columns of a dataset distributed as `di`: + +dstat(di, [1, 2, 3]) +# +# ## Manual work with the DistributedData.jl package +# +# We will first show how to use the general framework to compute per-cluster +# statistics. DistributedData.jl exports the `dmapreduce` function that can be +# used as a very effective basic building block for running such computations. +# For example, you can efficiently compute a distributed mean of all your data +# as such: + +dmapreduce(di, sum, +) / dmapreduce(di, length, +) + +# The parameters of `dmapreduce` are, in order: +# - `di`, the dataset +# - `sum` or `length`, an unary "map" function -- during the computation, each +# piece of distributed data is first _paralelly_ processed by this function +# - `+`, a binary "reduction" or "folding" function -- the pieces of information +# processed by the map function are successively joined in pairs using this +# function, until there is only a single result left. This final result is +# also what `dmapreduce` returns. +# +# Above example thus reads: Sum all data on all workers, add up the intermediate +# results, and divide the final number to the sum of all lengths of data on the +# workers. +# +# Column-wise mean (as produced by `dstat`) is slightly more useful; we only need +# to split the computation on columns: + +dmapreduce(di, d -> mapslices(sum, d, dims = 1), +) ./ dmapreduce(di, x->size(x, 1), +) + +# Finally, for distributed computation of per-cluster mean, the clustering +# information needs to be distributed as well (Fortunately, that is easy, +# because the distributed `assign` does exactly that). +# +# First, compute the clustering: +using GigaSOM +som = init(di, 10, 10, seed = 12345) +train(som, di) +clustering = assign(som, di) + +# Given a metaclustering (as produced in the [flow cytometry +# tutorial](02-flow-data.md), we can transform the clustering to SOM clusters to +# the clustering to metaclusters. (For simplicity, we use a random metaclustering +# here.) + +import Random +Random.seed!(12345); # set a seed for reproducibility of rand() + +meta_clusters = rand(1:5, 100); +dtransform(clustering, m -> meta_clusters[m]) + +# The computation is automatically run over the 2 distributed partitions of the +# dataset. +# +# To aggregate some statistic information about the clusters, we use a helper +# function `mapbuckets` which provides bucket-wise execution of any +# statistics-generating function, in a way very similar to `mapslices`. (In the +# example, we actually use `catmapbuckets` that concatenates the result into a +# nice array.) The following code produces a matrix of tuples `(sum, count)`, +# for separate clusters (in rows) and data columns (in columns): + +sums_counts = dmapreduce( + [di, clustering], + (d, clustering) -> DistributedData.catmapbuckets( + (_, clust) -> (sum(clust), length(clust)), + d, + 5, + clustering, + ), + (a, b) -> (((as, al), (bs, bl)) -> ((as+bs), (al+bl))).(a, b), +) + +# With a bit of extra programming, the gathered information can be aggregated +# to produce actual per-cluster means: +cluster_means = [sum/count for (sum, count) in sums_counts] + +# ## Convenience statistical functions +# +# Notably, several of the most used statistical functions are available in +# DistributedData.jl in a form that can cope with distributed data. +# +# For example, you can run a distributed median computation as such: +dmedian(di, [1, 2, 3, 4]) + +# In the hypercube dataset, the medians are slightly off-center because there +# is a lot of empty space between the clusters. + +# `dstat` function has a bucketed variant that can split the statistics among +# different clusters. This computes the per-cluster standard deviations of the +# dataset: + +(means, sds) = dstat_buckets(di, 5, clustering, [1, 2, 3, 4]) + +# In the result, we can count 4 "nice" clusters, and 6 clusters that span 2 of +# the original clusters, totally giving 16. (Hypercube validation succeeded!) +# +# A similar bucketed version is available for computation of medians: +dmedian_buckets(di, 5, clustering, [1, 2, 3, 4]) + +# Note that the cluster medians are similar to means, except for the cases when +# the cluster is formed by 2 actual data aggregations (e.g. on the second row), +# where medians dodge the empty space in the middle of the data: + +# ## Cleaning up + +# Finally, we can remove the temporary data from workers to create free memory +# for other analyses: +unscatter(clustering) diff --git a/docs/src/tutorials/basicUsage.md b/docs/src/tutorials/basicUsage.md deleted file mode 100644 index 888da529..00000000 --- a/docs/src/tutorials/basicUsage.md +++ /dev/null @@ -1,152 +0,0 @@ - -# Tutorial 1: Intro & basic usage - -For the installation of Julia or GigaSOM.jl please refer to the installation -instructions. - -## High-level overview - -GigaSOM provides functions that allow straightforward loading of FCS files into -matrices, preparing these matrices for analysis, and running SOM and related -function on the data. - -The main functions, listed by category: - -- [`loadFCS`](@ref) and [`loadFCSSet`](@ref) for loading the data -- [`getMetaData`](@ref) and [`getMarkerNames`](@ref) for accessing the - information about data columns stored in FCS files -- [`dselect`](@ref), [`dtransform_asinh`](@ref), [`dscale`](@ref) and similar - functions for transforming, scaling and preparing the data -- [`initGigaSOM`](@ref) for initializing the self-organizing map, - [`trainGigaSOM`](@ref) for running the SOM training, [`mapToGigaSOM`](@ref) - for classification using the trained SOM, and [`embedGigaSOM`](@ref) for - dimensionality reduction to 2D - -Multiprocessing is done using the `Distributed` package -- if you add -more "workers" using the `addprocs` function, GigaSOM will -automatically react to that situation, split the data among the processes and -run parallel versions of all algorithms. - -### Horizontal scaling - -While all functions work well on simple data matrices, the main aim of GigaSOM -is to let the users enjoy the cluster-computing resources. All functions also -work on data that are "scattered" among workers (i.e. each worker only holds a -portion of the data loaded in the memory). This dataset description is stored -in [`Dinfo`](@ref) structure. Most of the functions above in fact -accept the `Dinfo` as argument, and often return another -`Dinfo` that describes the scattered result. - -Most importantly, using `Dinfo` **prevents memory exhaustion at the -master node**, which is a critical feature required to handle huge datasets. - -You can always collect the scattered data back into a matrix (if it fits to -your RAM) with `gather_array`, and utilize many other functions to manipulate -it, including e.g. `dmapreduce` for easily running parallel computations, or -`dstore` for saving and restoring the dataset paralelly. - -## Minimal working example - -First, load GigaSOM: - -```julia -using GigaSOM -``` - -We will create a bit of randomly generated data for this purpose. This code -generates a 4D hypercube of size 10 with gaussian clusters at vertices: - -```julia -d = randn(10000,4) .+ rand(0:1, 10000, 4).*10 -``` - -The SOM (of size 20x20) is created and trained as such: - -```julia -som = initGigaSOM(d, 20, 20) -som = trainGigaSOM(som, d) -``` - -(Note that SOM initialization is randomized; if you want to get the same -results everytime, use e.g. `Random.seed!(1)`.) - -You can now see the SOM codebook (your numbers will vary): - -```julia -som.codes -``` - -``` -400×4 Array{Float64,2}: - -0.361681 -0.57191 0.140438 9.99224 - -1.111 1.60277 -0.209706 9.96805 - -1.23305 7.58148 -0.445886 9.76316 - -0.285692 9.80184 -1.12107 9.85507 - -0.197007 10.8793 -0.649294 9.89448 - -0.334737 11.0858 0.213889 9.93479 - 0.00282155 11.0725 0.718114 10.2714 - -0.333398 10.1315 1.14412 10.564 - -0.0124202 1.48128 8.35741 10.72 - -0.0084074 0.0150858 9.91007 11.5361 - ⋮ -``` - -This information can be used to categorize the dataset into clusters: - -```julia -mapToGigaSOM(som, d) -``` - -In the result, `index` is a cluster ID for the original datapoint from `d` at -the same row. -``` -10000×1 DataFrames.DataFrame -│ Row │ index │ -│ │ Int64 │ -├───────┼───────┤ -│ 1 │ 381 │ -│ 2 │ 178 │ -│ 3 │ 348 │ -│ 4 │ 379 │ -│ 5 │ 80 │ -│ 6 │ 146 │ -│ 7 │ 57 │ -⋮ -``` - -(As in the previous case, your numbers may differ.) - -Finally, you can use EmbedSOM dimensionality reduction to convert all -multidimensional points to 2D; which can eventually be used to create a -good-looking 2D scatterplot. -```julia -e = embedGigaSOM(som,d) -``` - -``` -10000×2 Array{Float64,2}: - 1.41575 18.5282 - 17.4483 7.4137 - 6.88243 17.5722 - 17.654 18.0348 - 17.594 3.15645 - 5.27181 8.61096 - 15.9708 2.8124 - 6.19637 9.05302 - 1.49358 7.19198 - 16.596 7.75608 - ⋮ -``` - -The 2D coordinates may be plotted by any standard plotting library. In the -following example we show how to do that with `Gadfly`: -```julia -Pkg.add("Gadfly") -Pkg.add("Cairo") -using Gadfly -import Cairo -draw(PNG("test.png",20cm,20cm), plot(x=e[:,1], y=e[:,2], color=d[:,1])) -``` - -In the resulting picture, you should be able to see all 16 gaussian clusters -colored by the first dimension in the original space. diff --git a/docs/src/tutorials/distributedProcessing.md b/docs/src/tutorials/distributedProcessing.md deleted file mode 100644 index 4a0c6aea..00000000 --- a/docs/src/tutorials/distributedProcessing.md +++ /dev/null @@ -1,185 +0,0 @@ - -# Tutorial 3: Distributed data processing and statistics - -If you can get the data on a single machine, computation of various statistics -can be performed using standard Julia functions. With large datasets that do -not fit on a computer, things get more complicated. Luckily, many statistics -and algorithms possess parallel, map-reduce-style implementations that can be -used to address this problem. - -For example, the [`dstat`](@ref) function computes sample statistics without -fetching all data to a single node. You can use it in a way similar to -aforementioned `dselect` and `dtransform_asinh`. The following code extracts -means and standard deviations from the first 3 columns of a dataset distributed -as `di`: -```julia -using DistributedData - -dstat(di, [1,2,3]) -``` - -## Manual work with the DistributedData.jl package - -We will first show how to use the general framework to compute per-cluster -statistics. DistributedData.jl exports the [`dmapreduce`](@ref) function that -can be used as a very effective basic building block for running such -computations. For example, you can efficiently compute a distributed mean of -all your data as such: -```julia -dmapreduce(di, sum, +) / dmapreduce(di, length, +) -``` - -The parameters of `dmapreduce` are, in order: - -- `di`, the dataset -- `sum` or `length`, an unary "map" function -- during the computation, each - piece of distributed data is first _paralelly_ processed by this function -- `+`, a binary "reduction" or "folding" function -- the pieces of information - processed by the map function are successively joined in pairs using this - function, until there is only a single result left. This final result is also - what `dmapreduce` returns. - -Above example thus reads: Sum all data on all workers, add up the intermediate -results, and divide the final number to the sum of all lengths of data on the -workers. - -Column-wise mean (as produced by `dstat`) is slightly more useful; we only need -to split the computation on columns: - -```julia -dmapreduce(di, d -> mapslices(sum, d, dims=1), +) ./ dmapreduce(di, x->size(x,1), +) -``` - -Finally, for distributed computation of per-cluster mean, the clustering -information needs to be distributed as well (Fortunately, that is easy, because -the distributed `mapToGigaSOM` does exactly that). - -First, compute the clustering: -```julia -mapping = mapToGigaSOM(som, di) -dtransform(mapping, m -> metaClusters[m]) -``` - -Now, the distributed computation is run on 2 scattered datasets. We employ a -helper function `mapbuckets` which provides bucket-wise execution of a -function, in a way very similar to `mapslices`. (In the example, we actually -use `catmapbuckets` that concatenates the result into a nice array.) The -following code produces a matrix of tuples `(sum, count)`, for separate -clusters (in rows) and data columns (in columns): - -```julia -sumscounts = dmapreduce([di, mapping], - (d, mapping) -> catmapbuckets( - (_,clData) -> (sum(clData), length(clData)), - d, 10, mapping), - (a,b) -> (((as,al),(bs,bl)) -> ((as+bs), (al+bl))).(a,b)) -``` - -``` -10×4 Array{Tuple{Float64,Int64},2}: - (5949.71, 1228) (-21.9789, 1228) (12231.3, 1228) (12303.1, 1228) - (6379.98, 1246) (12464.3, 1246) (12427.9, 1246) (12479.8, 1246) - (6513.41, 1294) (12968.8, 1294) (12960.7, 1294) (-28.1922, 1294) - (6312.37, 1236) (-26.7392, 1236) (6.74384, 1236) (12401.7, 1236) - (6395.73, 1285) (12867.7, 1285) (-52.653, 1285) (-26.9795, 1285) - (6229.72, 622) (10.7578, 622) (6200.1, 622) (0.882128, 622) - (6141.97, 612) (6078.56, 612) (45.9878, 612) (6079.3, 612) - (51.3709, 616) (23.4306, 616) (6117.53, 616) (1.15342, 616) - (6177.16, 1207) (-50.4624, 1207) (48.8023, 1207) (-5.549, 1207) - (8.56597, 654) (6536.1, 654) (-29.2208, 654) (6539.94, 654) -``` - -With a bit of Julia, this can be aggregated to actual per-cluster means: -```julia -clusterMeans = [ sum/count for (sum,count) in sumcounts ] -``` - -``` -10×4 Array{Float64,2}: - 4.84504 -0.0178982 9.96031 10.0188 - 5.12037 10.0034 9.97428 10.0159 - 5.03354 10.0223 10.016 -0.0217869 - 5.10709 -0.0216336 0.00545618 10.0337 - 4.97722 10.0138 -0.0409751 -0.0209958 - 10.0156 0.0172955 9.968 0.00141821 - 10.0359 9.93229 0.0751434 9.9335 - 0.0833944 0.0380366 9.93105 0.00187243 - 5.11778 -0.0418081 0.0404327 -0.00459735 - 0.0130978 9.99403 -0.0446802 9.99991 -``` - -Since we used the data from the hypercube dataset from the beginning of the -tutorial, you should be able to recognize several clusters that perfectly match -the hypercube vertices (although not all, because `k=10` is not enough to -capture all of the actual 16 existing clusters) - -Finally, we can remove the temporary data from workers to create free memory for other analyses: -```julia -unscatter(mapping) -``` - -## Convenience statistical functions - -Notably, several of the most used statistical functions are available in -DistributedData.jl in a form that can cope with distributed data. - -For example, you can run a distributed median computation as such: -```julia -dmedian(di, [1,2,3,4]) -``` - -In the hypercube dataset, the medians are slightly off-center because there is -a lot of empty space between the clusters: -``` -3-element Array{Float64,1}: - 6.947097488861494 - 7.934405685940568 - 7.069149844215707 - 2.558892109203585 -``` - -`dstat` function has a bucketed variant that can split the statistics among -different clusters. This computes the per-cluster standard deviations of the -dataset: - -```julia -dstat_buckets(di, 10, mapping, [1,2,3,4])[2] -``` - -In the result, we can count 4 "nice" clusters, and 6 clusters that span 2 of -the original clusters, totally giving 16. (Hypercube validation succeeded!) -``` -10×4 Array{Float64,2}: - 5.09089 0.997824 1.01815 0.980758 - 5.13971 1.02019 0.977637 1.00124 - 5.13209 0.974332 1.00058 0.99874 - 5.11529 0.998166 1.01825 1.01885 - 5.10542 1.01686 0.975993 0.991992 - 0.991075 0.993312 1.00667 1.05048 - 0.996443 1.02699 0.938742 0.98831 - 0.946917 0.989543 1.0056 0.999609 - 5.09963 1.00131 0.978803 0.984435 - 1.00892 0.998226 1.05538 0.994829 -``` - -A similar bucketed version is available for computation of medians: -```julia -dmedian_buckets(di, 10, mapping, [1,2,3,4]) -``` - -Note that the cluster medians are similar to means, except for the cases when -the cluster is formed by 2 actual data aggregations (e.g. on the second row), -where medians dodge the empty space in the middle of the data: -``` -10×4 Array{Float64,2}: - 1.97831 -0.0120118 9.98967 10.0161 - 7.99438 10.0263 9.9988 10.0033 - 3.27907 9.98728 10.0254 0.00444198 - 7.91739 -0.0623953 -0.0240277 10.0374 - 2.445 10.0101 -0.0471141 -0.0253346 - 10.0121 0.00935064 9.94992 0.0459787 - 10.0512 9.93359 0.0923141 9.91175 - 0.0675462 -0.0142712 9.93406 0.0343599 - 8.09972 -0.0217352 0.0575258 -0.010485 - -0.0183372 10.0392 -0.115253 10.0101 -``` diff --git a/docs/src/tutorials/processingFCSData.md b/docs/src/tutorials/processingFCSData.md deleted file mode 100644 index 7c891fc7..00000000 --- a/docs/src/tutorials/processingFCSData.md +++ /dev/null @@ -1,272 +0,0 @@ - -# Tutorial 2: Working with cytometry data - -You can load any FCS file using [`loadFCS`](@ref) function. For example, the -Levine dataset ([obtainable here](https://flowrepository.org/id/FR-FCM-ZZPH)) -may be loaded as such: - -``` -params, data = loadFCS("Levine_13dim.fcs") -``` - -`params` will now contain the list of FCS parameters; you can parse a lot of -interesting information from it using the [`getMetaData`](@ref) function: - -```julia -getMetaData(params) -``` - -``` -14×8 DataFrames.DataFrame. Omitted printing of 3 columns -│ Row │ E │ S │ N │ RMIN │ R │ -│ │ String │ String │ String │ String │ String │ -├─────┼────────┼────────┼────────┼───────────────────┼────────┤ -│ 1 │ 0,0 │ │ CD45 │ -2.03601189282714 │ 1024 │ -│ 2 │ 0,0 │ │ CD45RA │ -2.99700270621007 │ 1024 │ -│ 3 │ 0,0 │ │ CD19 │ -3.05850183816765 │ 1024 │ -│ 4 │ 0,0 │ │ CD11b │ -2.99956408593931 │ 1024 │ -│ 5 │ 0,0 │ │ CD4 │ -2.22860674361335 │ 1024 │ -│ 6 │ 0,0 │ │ CD8 │ -3.29174106765763 │ 1024 │ -│ 7 │ 0,0 │ │ CD34 │ -2.74278770893026 │ 1024 │ -│ 8 │ 0,0 │ │ CD20 │ -3.40866348184011 │ 1024 │ -│ 9 │ 0,0 │ │ CD33 │ -2.31371406643428 │ 1024 │ -│ 10 │ 0,0 │ │ CD123 │ -3.02624638359366 │ 1024 │ -│ 11 │ 0,0 │ │ CD38 │ -3.14752313833461 │ 1024 │ -│ 12 │ 0,0 │ │ CD90 │ -2.55305031846157 │ 1024 │ -│ 13 │ 0,0 │ │ CD3 │ -3.52459385416266 │ 1024 │ -│ 14 │ 0,0 │ │ label │ 1 │ 1024 │ -``` - -`data` is a matrix with cell expressions, one cell per row, one marker per -column. If you want to run SOM analysis on it, you can cluster and visualize it -just as in the previous tutorial, with one exception- we start with cutting off -the `label` column that contains `NaN` values: - -``` - -data = data[:,1:13] -som = initGigaSOM(data, 16, 16) -som = trainGigaSOM(som, data) -clusters = mapToGigaSOM(som, data) -e = embedGigaSOM(som, data) - -# ... save/plot results, etc... -``` - -## Work with distributed data - -Usual experiments produce multiple FCS files, and distributed or parallel -processing is very helpful in crunching through all the data. - -To load multiple FCS files, use [`loadFCSSet`](@ref). This function works well -in the "usual" single-process environment, but additionally it is designed to -handle situations when the data is too big to fit into memory, and attempts to -split them into available distributed workers workers. - -For the purpose of data distribution, you need to identify each dataset by an -unique **dataset name** that will be used for identifying your loaded data in -the cluster environment. The dataset name is a simple Julia symbols; basically -a variable name that is prefixed with a `:` colon. - -For example, we can load the Levine13 dataset as such: - -```julia -datainfo = loadFCSSet(:levine, ["Levine_13dim.fcs"]) -``` - -Expectably, if you have more files, just write their names into the array and -the function will handle the rest. - -The result `datainfo` carries informaton about your selected dataset name and -its distribution among the cluster. It can be used just as the "data" parameter -in all SOM-related functions again; e.g. as `trainGigaSOM(som, datainfo)`. - -The following example exploits the possibility to actually split the data, and -processes the Levine dataset parallelly on 2 workers: - -```julia -using Distributed -addprocs(2) # add any number of CPUs/tasks/workers you have available -@everywhere using GigaSOM # load GigaSOM also on the workers - -datainfo = loadFCSSet(:levine, ["Levine_13dim.fcs"]) # add more files as needed - -dselect(datainfo, Vector(1:13)) # select columns that contain expressions (column 14 contains labels) -som = initGigaSOM(datainfo, 20, 20) -som = trainGigaSOM(som, datainfo) -``` - -To prevent memory overload of the "master" computation node, the results of all -per-cell operations are also stored in distributed datainfo objects. In this -case, the following code does the embedding, but leaves the resulting data -safely scattered among the cluster: -```julia -e = embedGigaSOM(som, datainfo) -``` - -If you are sure you have enough RAM, you can collect the data to the master -node. (In case of the relatively small Levine13 dataset, you very probably have -the required 2.5MB of RAM, but there are many larger datasets.) -```julia -e = gather_array(e) -``` - -``` -167044×2 Array{Float64,2}: - 16.8251 11.5002 - 18.2608 12.884 - 12.0103 5.18401 - 18.381 12.3436 - 18.357 11.6622 - 14.8936 12.0897 - 17.6441 12.3652 - 17.8752 12.7206 - 17.301 11.0767 - 14.2055 12.2227 - ⋮ -``` - -## Working with larger datasets - -In this example we will use a subset of the Cytometry data [by Bodenmiller et -al.](https://doi.org/10.1038/nbt.2317). This data-set contains samples -from peripheral blood mononuclear cells (PBMCs) in unstimulated and stimulated -conditions for 8 healthy donors. - -10 cell surface markers (lineage markers) are used to identify different cell -populations. The dataset is described in two files: - -- `PBMC8_panel.xlsx` (with antigen names categorized as lineage markers and functional markers) -- `PBMC8_metadata.xlsx` (file names, sample IDs, condition IDs and patient IDs) - -### Download and prepare the dataset - -The example data can be downloaded from [imlspenticton.uzh.ch/robinson_lab/cytofWorkflow/](http://imlspenticton.uzh.ch/robinson_lab/cytofWorkflow/) - -You can fetch the files directly from within Julia: - -```julia -# fetch the required data for testing and download the zip archive and unzip it -dataFiles = ["PBMC8_metadata.xlsx", "PBMC8_panel.xlsx", "PBMC8_fcs_files.zip"] -for f in dataFiles - if !isfile(f) - download("http://imlspenticton.uzh.ch/robinson_lab/cytofWorkflow/"*f, f) - if occursin(".zip", f) - run(`unzip PBMC8_fcs_files.zip`) - end - end -end -``` - -The metadata is present in external files; we read it into a `DataFrame` and -extract information about FCS data columns from there. First, we read the -actual content using the XLSX package: - -```julia -using XLSX -md = GigaSOM.DataFrame(readtable("PBMC8_metadata.xlsx", "Sheet1", infer_eltypes=true)...) -panel = GigaSOM.DataFrame(readtable("PBMC8_panel.xlsx", "Sheet1", infer_eltypes=true)...) -``` - -After that, we can get the parameter structure from the first FCS files: -```julia -_, fcsParams = loadFCSHeader(md[1, :file_name]) -``` - -Continue with extracting marker names using the prepared functions: -```julia -_, fcsAntigens = getMarkerNames(getMetaData(fcsParams)) -``` - -Now, see which antigens we want to use (assume we want only the lineage markers): -```julia -antigens = panel[panel[:,:Lineage].==1, :Antigen] -``` - -Finally, it is often useful to make the names a bit more Julia-friendly and predictable: -```julia -cleanNames!(antigens) -cleanNames!(fcsAntigens) -``` - -### Load and prepare the data - -Now we have the vector of `fcsAntigens` that the FCS files store, and list of -`antigens` that we want to analyze. We continue by loading the data, reducing -it to the desired antigens and transforming it a bit: - -```julia -di = loadFCSSet(:pbmc8, md[:,:file_name]) -``` - -(If data distribution and parallelization is required, you must add parallel -workers using `addprocs` **before** this step.) - -Now that the data is loaded, let's prepare them a bit by reducing to actual -interesting columns, transformation and scaling: - -```julia -# select only the columns that correspond to the lineage antigens we have prepared before -dselect(di, fcsAntigens, antigens) - -cols = Vector(1:length(antigens)) # shortcut for "all rows" - -# perform asinh transformation on all data in the dataset, '5' is the cofactor for the transformation -dtransform_asinh(di, cols, 5) - -# normalize all dataset columns to mean=0 sdev=1 -dscale(di, cols) -``` - -### Create a Self Organizing MAP (SOM) - -With the data prepared, running the SOM algorithm is straightforward: - -```julia -# randomly initialize the SOM -som = initGigaSOM(di, 16, 16) - -# train the SOM for 20 epochs (10 is default, but nothing will happen if the -# epochs are slightly overdone) -som = trainGigaSOM(som, di, epochs = 20) -``` - -Finally, calculate the clustering: - -```julia -somClusters = mapToGigaSOM(som, di) -``` - -### FlowSOM-style metaclustering - -One disadvantage of SOMs is that they output a large amount of small -clusters that are relatively hard to interpret manually. FlowSOM improved that -situation by running a "clustering on clusters" (metaclustering) that address -the problem. - -In this example, we reduce the original 256 small clusters from 16x16 SOM to -only 10 "metaclusters", using the standard hierarchical clustering: - -```julia -using Clustering -import Distances -metaClusters = - cutree(k=10, - hclust(linkage=:average, - GigaSOM.distMatrix(Distances.Euclidean())(som.codes))) -``` - -The `metaClusters` represent membership of the SOM codes in cluster; these can -be expanded to membership of all cells using [`mapToGigaSOM`](@ref): - -```julia -mapping = gather_array(mapToGigaSOM(som, di), free=true) -clusters = metaClusters[mapping] -``` - -`clusters` now contain an integer from `1` to `10` with a classification of -each cell in the dataset. - -(The argument `free=true` of `gather_array` automatically removes the -distributed data from workers after collecting, which saves their memory for -other datasets.) diff --git a/gigasom.def b/gigasom.def deleted file mode 100644 index b67ee0ca..00000000 --- a/gigasom.def +++ /dev/null @@ -1,22 +0,0 @@ -Bootstrap: library -From: cylon-x/lcsb-biocore/julia-base:latest - -%setup - export GIT_WORK_TREE="${SINGULARITY_ROOTFS}/project/" - export GIT_DIR="`pwd`/.git" - mkdir -p "${GIT_WORK_TREE}" - git checkout -f - -%post - export JULIA_DEPOT_PATH=/user/.julia - export PATH=/opt/julia/bin:$PATH - - cd project - julia --project -e 'import Pkg; Pkg.instantiate(); Pkg.build(); Pkg.precompile();' - - echo 'using GigaSOM' > /project/.startup.jl - - chmod -R a+rX $JULIA_DEPOT_PATH - -%runscript - julia --banner=no --project=/project --compile=min -i /project/.startup.jl diff --git a/src/GigaSOM.jl b/src/GigaSOM.jl index ad9466fe..3514837f 100644 --- a/src/GigaSOM.jl +++ b/src/GigaSOM.jl @@ -1,11 +1,29 @@ """ -Main module for `GigaSOM.jl` - Huge-scale, high-performance flow cytometry clustering + module GigaSOM -The documentation is here: http://LCSB-BioCore.github.io/GigaSOM.jl -""" +GigaSOM.jl allows painless analysis of huge-scale flow-cytometry datasets, such +as from large clinical studies. It can be viewed as a work-alike of FlowSOM +package (for R), suitable for loading billions of cells and running the +analyses in parallel on distributed computer clusters, in order to gain speed. + +Most importantly, GigaSOM.jl scales horizontally -- data volume limitations and +memory limitations can be solved just by adding more computers to the cluster. +That makes it extremely easy to exploit HPC environments, which are becoming +increasingly common in computational biology. + +### Features + +- Horizontal scalability to literal giga-scale datasets (``10^9`` cells!) +- HPC-ready, with support for e.g. Slurm +- Standard support for distributed loading, scaling and transforming the FCS3 files +- Batch-SOM based GigaSOM algorithm for clustering +- EmbedSOM for visualizations +""" module GigaSOM +using DocStringExtensions + using CSV using DataFrames using Distances @@ -14,58 +32,18 @@ using DistributedData using Distributions using FCSFiles using FileIO -using DistributedArrays using NearestNeighbors using Serialization using StableRNGs -include("base/structs.jl") - -include("base/dataops.jl") -include("base/trainutils.jl") - -include("analysis/core.jl") -include("analysis/embedding.jl") - -include("io/input.jl") -include("io/process.jl") -include("io/splitting.jl") - -#core -export initGigaSOM, trainGigaSOM, mapToGigaSOM - -#trainutils -export linearRadius, expRadius, gaussianKernel, bubbleKernel, thresholdKernel, distMatrix - -#embedding -export embedGigaSOM - -# structs -export Som - -#io/input -export readFlowset, - readFlowFrame, - loadFCS, - loadFCSHeader, - getFCSSize, - loadFCSSizes, - loadFCSSet, - selectFCSColumns, - distributeFCSFileVector, - distributeFileVector, - getCSVSize, - loadCSV, - loadCSVSizes, - loadCSVSet - -#io/splitting -export slicesof, vcollectSlice, collectSlice +include("structs.jl") -#io/process -export cleanNames!, getMetaData, getMarkerNames +include("som.jl") +include("som_utils.jl") +include("embed.jl") -#dataops (higher-level operations on data) -export dtransform_asinh +include("load.jl") +include("split.jl") +include("data_utils.jl") end # module diff --git a/src/base/dataops.jl b/src/base/dataops.jl deleted file mode 100644 index e9d8fee3..00000000 --- a/src/base/dataops.jl +++ /dev/null @@ -1,8 +0,0 @@ -""" - dtransform_asinh(dInfo::Dinfo, columns::Vector{Int}, cofactor=5) - -Transform columns of the dataset by asinh transformation with `cofactor`. -""" -function dtransform_asinh(dInfo::Dinfo, columns::Vector{Int}, cofactor = 5) - dapply_cols(dInfo, (v, _) -> asinh.(v ./ cofactor), columns) -end diff --git a/src/io/process.jl b/src/data_utils.jl similarity index 80% rename from src/io/process.jl rename to src/data_utils.jl index 52a71eb5..c9416f74 100644 --- a/src/io/process.jl +++ b/src/data_utils.jl @@ -1,5 +1,5 @@ """ - cleanNames!(mydata::Vector{String}) +$(TYPEDSIGNATURES) Replaces problematic characters in column names, avoids duplicate names, and prefixes an '_' if the name starts with a number. @@ -7,7 +7,7 @@ prefixes an '_' if the name starts with a number. # Arguments: - `mydata`: vector of names (gets modified) """ -function cleanNames!(mydata::Vector{String}) +function clean_names!(mydata::Vector{String}) # replace problematic characters, # put "_" in front of colname in case it starts with a number # avoid duplicate names (add suffixes _2, _3, ...) @@ -29,14 +29,14 @@ function cleanNames!(mydata::Vector{String}) end end +export clean_names! + """ - getMetaData(f) -Collect the meta data information in a more user friendly format. +$(TYPEDSIGNATURES) -# Arguments: -- `f`: input structure with `.params` and `.data` fields +Collect the meta data information in a more user friendly format. """ -function getMetaData(meta::Dict{String,String})::DataFrame +function fcs_column_metadata(meta::Dict{String,String})::DataFrame # declarations and initializations metaKeys = keys(meta) @@ -77,14 +77,16 @@ function getMetaData(meta::Dict{String,String})::DataFrame return df end +export fcs_column_metadata + """ - getMarkerNames(meta::DataFrame)::Tuple{Vector{String}, Vector{String}} +$(TYPEDSIGNATURES) Extract suitable raw names (useful for selecting columns) and pretty readable names (useful for humans) from FCS file metadata. """ -function getMarkerNames(meta::DataFrame)::Tuple{Vector{String},Vector{String}} +function fcs_metadata_marker_names(meta::DataFrame)::Tuple{Vector{String},Vector{String}} orig = Array{String}(meta[:, :N]) nice = copy(orig) if hasproperty(meta, :S) @@ -97,9 +99,10 @@ function getMarkerNames(meta::DataFrame)::Tuple{Vector{String},Vector{String}} return (orig, nice) end +export fcs_metadata_marker_names """ - compensate!(data::Matrix{Float64}, spillover::Matrix{Float64}, cols::Vector{Int}) +$(TYPEDSIGNATURES) Apply a compensation matrix in `spillover` (the individual columns of which describe, in order, the spillover of `cols` in `data`) to the matrix `data` @@ -109,12 +112,14 @@ function compensate!(data::Matrix{Float64}, spillover::Matrix{Float64}, cols::Ve data[:, cols] = data[:, cols] * inv(spillover) end +export compensate! + """ - parseSpillover(str::String)::Union{Tuple{Vector{String},Matrix{Float64}}, Nothing} +$(TYPEDSIGNATURES) Parses the spillover matrix from the string from FCS parameter value. """ -function parseSpillover(str::String)::Tuple{Vector{String},Matrix{Float64}} +function parse_fcs_spillover(str::String)::Tuple{Vector{String},Matrix{Float64}} fields = split(str, ',') n = parse(Int, fields[1]) if length(fields) != 1 + n + n * n @@ -129,20 +134,33 @@ function parseSpillover(str::String)::Tuple{Vector{String},Matrix{Float64}} end """ - getSpillover(params::Dict{String, String})::Union{Tuple{Vector{String},Matrix{Float64}}, Nothing} +$(TYPEDSIGNATURES) Get a spillover matrix from FCS `params`. Returns a pair with description of columns to be applied, and with the actual spillover matrix. Returns `nothing` in case spillover is not present. """ -function getSpillover( +function fcs_spillover( params::Dict{String,String}, )::Union{Tuple{Vector{String},Matrix{Float64}},Nothing} spillNames = ["\$SPILL", "\$SPILLOVER", "SPILL", "SPILLOVER"] for i in spillNames if in(i, keys(params)) - return parseSpillover(params[i]) + return parse_fcs_spillover(params[i]) end end return nothing end + +export fcs_spillover + +""" +$(TYPEDSIGNATURES) + +Transform columns of the dataset by asinh transformation with `cofactor`. +""" +function dtransform_asinh(dInfo::Dinfo, columns::Vector{Int}, cofactor = 5) + dapply_cols(dInfo, (v, _) -> asinh.(v ./ cofactor), columns) +end + +export dtransform_asinh diff --git a/src/analysis/embedding.jl b/src/embed.jl similarity index 86% rename from src/analysis/embedding.jl rename to src/embed.jl index 28d35e2e..477c8b83 100644 --- a/src/analysis/embedding.jl +++ b/src/embed.jl @@ -1,13 +1,5 @@ """ - embedGigaSOM(som::GigaSOM.Som, - dInfo::Dinfo; - knnTreeFun = BruteTree, - metric = Euclidean(), - k::Int64=0, - adjust::Float64=1.0, - smooth::Float64=0.0, - m::Float64=10.0, - output::Symbol=tmp_symbol(dInfo))::Dinfo +$(TYPEDSIGNATURES) Return a data frame with X,Y coordinates of EmbedSOM projection of the data. @@ -28,8 +20,8 @@ Return a data frame with X,Y coordinates of EmbedSOM projection of the data. Data must have the same number of dimensions as the training dataset, and must be normalized using the same parameters. """ -function embedGigaSOM( - som::GigaSOM.Som, +function embed( + som::GigaSOM.SOM, dInfo::Dinfo; knnTreeFun = BruteTree, metric = Euclidean(), @@ -66,24 +58,17 @@ function embedGigaSOM( end """ - embedGigaSOM(som::GigaSOM.Som, - data; - knnTreeFun = BruteTree, - metric = Euclidean(), - k::Int64=0, - adjust::Float64=1.0, - smooth::Float64=0.0, - m::Float64=10.0) - -Overload of `embedGigaSOM` for simple DataFrames and matrices. This slices the +$(TYPEDSIGNATURES) + +Overload of `embed` for simple DataFrames and matrices. This slices the data using `DistributedArrays`, sends them the workers, and runs normal -`embedGigaSOM`. All data is properly `unscatter`d after the computation. +`embed`. All data is properly `unscatter`d after the computation. # Examples: Produce a 2-column matrix with 2D cell coordinates: ``` -e = embedGigaSOM(som, data) +e = embed(som, data) ``` Plot the result using 2D histogram from Gadfly: @@ -94,8 +79,8 @@ draw(PNG("output.png",20cm,20cm), Geom.histogram2d(xbincount=200, ybincount=200))) ``` """ -function embedGigaSOM( - som::GigaSOM.Som, +function embed( + som::GigaSOM.SOM, data; knnTreeFun = BruteTree, metric = Euclidean(), @@ -108,7 +93,7 @@ function embedGigaSOM( data = Matrix{Float64}(data) dInfo = scatter_array(:GigaSOMembeddingDataVar, data, workers()) - rInfo = embedGigaSOM( + rInfo = embed( som, dInfo, knnTreeFun = knnTreeFun, @@ -125,19 +110,13 @@ function embedGigaSOM( end """ - embedGigaSOM_internal(som::GigaSOM.Som, - data::Matrix{Float64}, - tree, - k::Int64, - adjust::Float64, - boost::Float64, - m::Float64) +$(TYPEDSIGNATURES) Internal function to compute parts of the embedding on a prepared kNN-tree structure (`tree`) and `smooth` converted to `boost`. """ function embedGigaSOM_internal( - som::GigaSOM.Som, + som::GigaSOM.SOM, data::Matrix{Float64}, tree, k::Int64, @@ -285,3 +264,5 @@ function embedGigaSOM_internal( return e end + +export embed diff --git a/src/io/input.jl b/src/load.jl similarity index 63% rename from src/io/input.jl rename to src/load.jl index 7e8901a0..4dba0c2f 100644 --- a/src/io/input.jl +++ b/src/load.jl @@ -1,9 +1,9 @@ """ - loadFCSHeader(fn::String)::Tuple{Vector{Int}, Dict{String,String}} +$(TYPEDSIGNATURES) Efficiently extract data offsets and keyword dictionary from an FCS file. """ -function loadFCSHeader(fn::String)::Tuple{Vector{Int},Dict{String,String}} +function read_fcs_header(fn::String)::Tuple{Vector{Int},Dict{String,String}} open(fn) do io offsets = FCSFiles.parse_header(io) params = FCSFiles.parse_text(io, offsets[1], offsets[2]) @@ -13,12 +13,12 @@ function loadFCSHeader(fn::String)::Tuple{Vector{Int},Dict{String,String}} end """ - getFCSSize(offsets, params)::Tuple{Int,Int} +$(TYPEDSIGNATURES) Convert the offsets and keywords from an FCS file to cell and parameter count, respectively. """ -function getFCSSize(offsets, params)::Tuple{Int,Int} +function read_fcs_size(offsets, params)::Tuple{Int,Int} nData = parse(Int, params["\$TOT"]) nParams = parse(Int, params["\$PAR"]) @@ -46,21 +46,21 @@ function getFCSSize(offsets, params)::Tuple{Int,Int} end """ - loadFCSSizes(fns::Vector{String}) +$(TYPEDSIGNATURES) Load cell counts in many FCS files at once. Useful as input for `slicesof`. """ -function loadFCSSizes(fns::Vector{String})::Vector{Int} +function read_fcs_sizes(fns::Vector{String})::Vector{Int} [( begin - o, s = loadFCSHeader(fn) - getFCSSize(o, s)[1] + o, s = read_fcs_header(fn) + read_fcs_size(o, s)[1] end ) for fn in fns] end """ - loadFCS(fn::String; applyCompensation::Bool=true)::Tuple{Dict{String,String}, Matrix{Float64}} +$(TYPEDSIGNATURES) Read a FCS file. Return a tuple that contains in order: @@ -73,15 +73,15 @@ If `applyCompensation` is set, the function parses and retrieves a spillover matrix (if any valid keyword in the FCS is found that would contain it) and applies it to compensate the data. """ -function loadFCS( +function load_fcs( fn::String; applyCompensation::Bool = true, )::Tuple{Dict{String,String},Matrix{Float64}} fcs = FileIO.load(fn) - meta = getMetaData(fcs.params) + meta = fcs_column_metadata(fcs.params) data = hcat(map(x -> Vector{Float64}(fcs.data[x]), meta[:, :N])...) if applyCompensation - spill = getSpillover(fcs.params) + spill = fcs_spillover(fcs.params) if spill != nothing names, mtx = spill cols = indexin(names, meta[:, :N]) @@ -95,19 +95,21 @@ function loadFCS( return (fcs.params, data) end +export load_fcs + """ - loadFCSSet(name::Symbol, fns::Vector{String}, pids=workers(); applyCompensation=true, postLoad=(d,i)->d)::Dinfo +$(TYPEDSIGNATURES) This runs the FCS loading machinery in a distributed way, so that the files `fns` (with full path) are sliced into equal parts and saved as a distributed variable `name` on workers specified by `pids`. -`applyCompensation` is passed to loadFCS function. +`applyCompensation` is passed to load_fcs function. See `slicesof` for description of the slicing. `postLoad` is applied to the loaded FCS file data (and the index) -- use this -function to e.g. filter out certain columns right on loading, using `selectFCSColumns`. +function to e.g. filter out certain columns right on loading, using `select_fcs_columns`. The loaded dataset can be manipulated by the distributed functions, e.g. - `dselect` for removing columns @@ -115,24 +117,24 @@ The loaded dataset can be manipulated by the distributed functions, e.g. - `dtransform_asinh` (and others) for transformation - etc. """ -function loadFCSSet( +function load_fcs_distributed( name::Symbol, fns::Vector{String}, pids = workers(); applyCompensation = true, postLoad = (d, i) -> d, )::Dinfo - slices = slicesof(loadFCSSizes(fns), length(pids)) + slices = slicesof(read_fcs_sizes(fns), length(pids)) dmap( slices, (slice) -> Base.eval( Main, :( begin - $name = vcollectSlice( + $name = GigaSOM.vcollect_slice( (i) -> last( $postLoad( - loadFCS($fns[i]; applyCompensation = $applyCompensation), + load_fcs($fns[i]; applyCompensation = $applyCompensation), i, ), ), @@ -147,18 +149,20 @@ function loadFCSSet( return Dinfo(name, pids) end +export load_fcs_distributed + """ - selectFCSColumns(selectColnames::Vector{String}) +$(TYPEDSIGNATURES) -Return a function useful with `loadFCSSet`, which loads only the specified -(prettified) column names from the FCS files. Use `getMetaData`, -`getMarkerNames` and `cleanNames!` to retrieve the usable column names for a +Return a function useful with `load_fcs_distributed`, which loads only the specified +(prettified) column names from the FCS files. Use `fcs_column_metadata`, +`fcs_metadata_marker_names` and `clean_names!` to retrieve the usable column names for a FCS. """ -function selectFCSColumns(selectColnames::Vector{String}) +function select_fcs_columns(selectColnames::Vector{String}) ((metadata, data), idx) -> begin - _, names = getMarkerNames(getMetaData(metadata)) - cleanNames!(names) + _, names = fcs_metadata_marker_names(fcs_column_metadata(metadata)) + clean_names!(names) colIdxs = indexin(selectColnames, names) if any(colIdxs .== nothing) @error "Some columns were not found" @@ -168,27 +172,33 @@ function selectFCSColumns(selectColnames::Vector{String}) end end +export select_fcs_columns + """ - distributeFCSFileVector(name::Symbol, fns::Vector{String}, pids=workers())::Dinfo +$(TYPEDSIGNATURES) Distribute a vector of integers among the workers that describes which file from `fns` the cell comes from. Useful for producing per-file statistics. The vector is saved on workers specified by `pids` as a distributed variable `name`. """ -function distributeFCSFileVector(name::Symbol, fns::Vector{String}, pids = workers())::Dinfo - sizes = loadFCSSizes(fns) +function fcs_filevector_distribute( + name::Symbol, + fns::Vector{String}, + pids = workers(), +)::Dinfo + sizes = read_fcs_sizes(fns) slices = slicesof(sizes, length(pids)) - return distributeFileVector(name, sizes, slices, pids) + return filevector_distribute(name, sizes, slices, pids) end """ - distributeFileVector(name::Symbol, sizes::Vector{Int}, slices::Vector{Tuple{Int,Int,Int,Int}}, pids=workers())::Dinfo +$(TYPEDSIGNATURES) -Generalized version of `distributeFCSFileVector` that produces the integer +Generalized version of `fcs_filevector_distribute` that produces the integer vector from any `sizes` and `slices`. """ -function distributeFileVector( +function filevector_distribute( name::Symbol, sizes::Vector{Int}, slices::Vector{Tuple{Int,Int,Int,Int}}, @@ -196,24 +206,26 @@ function distributeFileVector( )::Dinfo dmap( slices, - (slice) -> - Base.eval(Main, :($name = collectSlice((i) -> fill(i, $sizes[i]), $slice))), + (slice) -> Base.eval( + Main, + :($name = GigaSOM.collect_slice((i) -> fill(i, $sizes[i]), $slice)), + ), pids, ) return Dinfo(name, pids) end """ - function getCSVSize(fn::String; args...)::Tuple{Int,Int} +$(TYPEDSIGNATURES) Read the dimensions (number of rows and columns, respectively) from a CSV file `fn`. `args` are passed to function `CSV.file`. # Example - getCSVSize("test.csv", header=false) + read_csv_size("test.csv", header=false) """ -function getCSVSize(fn::String; args...)::Tuple{Int,Int} +function read_csv_size(fn::String; args...)::Tuple{Int,Int} n = 0 k = 0 # ideally, this will not try to load the whole CSV in the memory @@ -227,53 +239,49 @@ function getCSVSize(fn::String; args...)::Tuple{Int,Int} end """ - function loadCSVSizes(fns::Vector{String}; args...)::Vector{Int} +$(TYPEDSIGNATURES) Determine number of rows in a list of CSV files (passed as `fns`). Equivalent -to `loadFCSSizes`. +to `read_fcs_sizes`. """ -function loadCSVSizes(fns::Vector{String}; args...)::Vector{Int} - [getCSVSize(fn, types = Float64; args...)[1] for fn in fns] +function read_csv_sizes(fns::Vector{String}; args...)::Vector{Int} + [read_csv_size(fn, types = Float64; args...)[1] for fn in fns] end """ - function loadCSV(fn::String; args...)::Matrix{Float64} +$(TYPEDSIGNATURES) -CSV equivalent of `loadFCS`. The metadata (header, column names) are not +CSV equivalent of `load_fcs`. The metadata (header, column names) are not extracted. `args` are passed to `CSV.read`. """ -function loadCSV(fn::String; args...)::Matrix{Float64} +function load_csv(fn::String; args...)::Matrix{Float64} CSV.read(fn, DataFrame, types = Float64; args...) |> Matrix{Float64} end +export load_csv + """ - function loadCSVSet( - name::Symbol, - fns::Vector{String}, - pids = workers(); - postLoad = (d, i) -> d, - csvargs..., - )::Dinfo - -CSV equivalent of `loadFCSSet`. `csvargs` are passed as keyword arguments to +$(TYPEDSIGNATURES) + +CSV equivalent of `load_fcs_distributed`. `csvargs` are passed as keyword arguments to CSV-loading functions. """ -function loadCSVSet( +function load_csv_distributed( name::Symbol, fns::Vector{String}, pids = workers(); postLoad = (d, i) -> d, csvargs..., )::Dinfo - slices = slicesof(loadCSVSizes(fns; csvargs...), length(pids)) + slices = slicesof(read_csv_sizes(fns; csvargs...), length(pids)) dmap( slices, (slice) -> Base.eval( Main, :( begin - $name = vcollectSlice( - (i) -> $postLoad(loadCSV($fns[i]; $csvargs...), i), + $name = GigaSOM.vcollect_slice( + (i) -> $postLoad(load_csv($fns[i]; $csvargs...), i), $slice, ) nothing @@ -284,3 +292,5 @@ function loadCSVSet( ) return Dinfo(name, pids) end + +export load_csv_distributed diff --git a/src/analysis/core.jl b/src/som.jl similarity index 61% rename from src/analysis/core.jl rename to src/som.jl index 83f40cda..3bbfd17f 100644 --- a/src/analysis/core.jl +++ b/src/som.jl @@ -1,15 +1,15 @@ """ - initGigaSOM(data, args...) +$(TYPEDSIGNATURES) Initializes a SOM by random selection from the training data. A generic overload that works for matrices and DataFrames that can be coerced to `Matrix{Float64}`. Other arguments are passed to the data-independent -`initGigaSOM`. +`init`. Arguments: - `data`: matrix of data for running the initialization """ -function initGigaSOM(data::Union{Matrix,DataFrame}, args...; kwargs...) +function init(data::Union{Matrix,DataFrame}, args...; kwargs...) d = Matrix{Float64}(data) @@ -17,33 +17,28 @@ function initGigaSOM(data::Union{Matrix,DataFrame}, args...; kwargs...) means = [sum(d[:, i]) / n for i = 1:ncol] sdevs = [sqrt(sum((d[:, i] .- means[i]) .^ 2.0) / n) for i = 1:ncol] - return initGigaSOM(ncol, means, sdevs, args...; kwargs...) + return init(ncol, means, sdevs, args...; kwargs...) end """ - function initGigaSOM(data::Dinfo, - xdim::Int64, ydim::Int64 = xdim; - seed=rand(Int), rng=StableRNG(seed)) +$(TYPEDSIGNATURES) -`initGigaSOM` overload for working with distributed-style `Dinfo` +`init` overload for working with distributed-style `Dinfo` data. The rest of the arguments is passed to the data-independent -`initGigaSOM`. +`init`. Arguments: - `data`: a `Dinfo` object with the distributed dataset matrix """ -function initGigaSOM(data::Dinfo, args...; kwargs...) +function init(data::Dinfo, args...; kwargs...) ncol = get_val_from(data.workers[1], :(size($(data.val))[2])) (means, sdevs) = dstat(data, Vector(1:ncol)) - initGigaSOM(ncol, means, sdevs, args...; kwargs...) + init(ncol, means, sdevs, args...; kwargs...) end """ - function initGigaSOM(ncol::Int64, - means::Vector{Float64}, sdevs::Vector{Float64}, - xdim::Int64, ydim::Int64 = xdim; - seed = rand(Int), rng = StableRNG(seed)) +$(TYPEDSIGNATURES) Generate a stable random initial SOM with the random distribution that matches the parameters. @@ -54,9 +49,9 @@ Arguments: - `seed`: a seed (defaults to random seed from the current default random generator - `rng`: a random number generator to be used (defaults to a `StableRNG` initialized with the `seed`) -Returns: a new `Som` structure +Returns: a new `SOM` structure """ -function initGigaSOM( +function init( ncol::Int64, means::Vector{Float64}, sdevs::Vector{Float64}, @@ -67,7 +62,7 @@ function initGigaSOM( ) numCodes = xdim * ydim - grid = gridRectangular(xdim, ydim) + grid = grid_rectangular(xdim, ydim) # Initialize with an unbiased random gaussian with same mean/sd as the data # in each dimension @@ -77,51 +72,40 @@ function initGigaSOM( codes[:, col] .+= means[col] end - return Som(codes = codes, xdim = xdim, ydim = ydim, grid = grid) + return SOM(codes = codes, xdim = xdim, ydim = ydim, grid = grid) end +export init """ - trainGigaSOM( - som::Som, - dInfo::Dinfo; - kernelFun::Function = gaussianKernel, - metric = Euclidean(), - somDistFun = distMatrix(Chebyshev()), - knnTreeFun = BruteTree, - rStart = 0.0, - rFinal = 0.1, - radiusFun = expRadius(-5.0), - epochs = 20, - eachEpoch = (e, r, som) -> nothing, - ) +$(TYPEDSIGNATURES) # Arguments: -- `som`: object of type Som with an initialised som +- `som`: object of type SOM with an initialised som - `dInfo`: `Dinfo` object that describes a loaded dataset -- `kernelFun::function`: optional distance kernel; one of (`bubbleKernel, gaussianKernel`) - default is `gaussianKernel` +- `kernelFun::function`: optional distance kernel; one of (`kernel_bubble, kernel_gaussian`) + default is `kernel_gaussian` - `metric`: Passed as metric argument to the KNN-tree constructor - `somDistFun`: Function for computing the distances in the SOM map - `knnTreeFun`: Constructor of the KNN-tree (e.g. from NearestNeighbors package) - `rStart`: optional training radius. If zero (default), it is computed from the SOM grid size. - `rFinal`: target radius at the last epoch, defaults to 0.1 -- `radiusFun`: Function that generates radius decay, e.g. `linearRadius` or `expRadius(10.0)` +- `radiusFun`: Function that generates radius decay, e.g. `radius_linear` or `radius_exp(10.0)` - `epochs`: number of SOM training iterations (default 10) - `eachEpoch`: a function to call back after each epoch, accepting arguments `(epochNumber, radius, som)`. For simplicity, this gets additionally called once before the first epoch, with `epochNumber` set to zero. """ -function trainGigaSOM( - som::Som, +function train( + som::SOM, dInfo::Dinfo; - kernelFun::Function = gaussianKernel, + kernelFun::Function = kernel_gaussian, metric = Euclidean(), - somDistFun = distMatrix(Chebyshev()), + somDistFun = distance_matrix(Chebyshev()), knnTreeFun = BruteTree, rStart = 0.0, rFinal = 0.1, - radiusFun = expRadius(-5.0), + radiusFun = radius_exp(-5.0), epochs = 20, eachEpoch = (e, r, som) -> nothing, ) @@ -143,7 +127,7 @@ function trainGigaSOM( for epoch = 1:epochs @debug "Epoch $epoch..." - numerator, denominator = distributedEpoch( + numerator, denominator = run_epoch_distributed( dInfo, result_som.codes, knnTreeFun(Array{Float64,2}(transpose(result_som.codes)), metric), @@ -166,27 +150,27 @@ function trainGigaSOM( end """ - trainGigaSOM(som::Som, train; - kwargs...) +$(TYPEDSIGNATURES) -Overload of `trainGigaSOM` for simple DataFrames and matrices. This slices the -data, distributes them to the workers, and runs normal `trainGigaSOM`. Data is +Overload of `train` for simple DataFrames and matrices. This slices the +data, distributes them to the workers, and runs normal `train`. Data is `unscatter`d after the computation. """ -function trainGigaSOM(som::Som, train; kwargs...) +function train(som::SOM, train_data; kwargs...) - train = Matrix{Float64}(train) + train_data = Matrix{Float64}(train_data) #this slices the data into parts and and sends them to workers - dInfo = scatter_array(:GigaSOMtrainDataVar, train, workers()) - som_res = trainGigaSOM(som, dInfo; kwargs...) + dInfo = scatter_array(:GigaSOMtrainDataVar, train_data, workers()) + som_res = train(som, dInfo; kwargs...) unscatter(dInfo) return som_res end +export train """ - doEpoch(x::Array{Float64, 2}, codes::Array{Float64, 2}, tree) +$(TYPEDSIGNATURES) vectors and the adjustment in radius after each epoch. @@ -195,7 +179,7 @@ vectors and the adjustment in radius after each epoch. - `codes`: Codebook - `tree`: knn-compatible tree built upon the codes """ -function doEpoch(x::Array{Float64,2}, codes::Array{Float64,2}, tree) +function run_epoch(x::Array{Float64,2}, codes::Array{Float64,2}, tree) # initialise numerator and denominator with 0's sumNumerator = zeros(Float64, size(codes)) @@ -215,24 +199,21 @@ function doEpoch(x::Array{Float64,2}, codes::Array{Float64,2}, tree) end """ - distributedEpoch(dInfo::Dinfo, codes::Matrix{Float64}, tree) +$(TYPEDSIGNATURES) -Execute the `doEpoch` in parallel on workers described by `dInfo` and collect +Execute the `run_epoch` in parallel on workers described by `dInfo` and collect the results. Returns pair of numerator and denominator matrices. """ -function distributedEpoch(dInfo::Dinfo, codes::Matrix{Float64}, tree) +function run_epoch_distributed(dInfo::Dinfo, codes::Matrix{Float64}, tree) return dmapreduce( dInfo, - (data) -> doEpoch(data, codes, tree), + (data) -> run_epoch(data, codes, tree), ((n1, d1), (n2, d2)) -> (n1 + n2, d1 + d2), ) end - """ - mapToGigaSOM(som::Som, dInfo::Dinfo; - knnTreeFun = BruteTree, metric = Euclidean(), - output::Symbol=tmp_symbol(dInfo)::Dinfo +$(TYPEDSIGNATURES) Compute the index of the BMU for each row of the input data. @@ -246,8 +227,8 @@ Compute the index of the BMU for each row of the input data. Data must have the same number of dimensions as the training dataset and will be normalised with the same parameters. """ -function mapToGigaSOM( - som::Som, +function assign( + som::SOM, dInfo::Dinfo; knnTreeFun = BruteTree, metric = Euclidean(), @@ -260,14 +241,13 @@ function mapToGigaSOM( end """ - mapToGigaSOM(som::Som, data; - knnTreeFun = BruteTree, - metric = Euclidean()) -Overload of `mapToGigaSOM` for simple DataFrames and matrices. This slices the -data using `DistributedArrays`, sends them the workers, and runs normal -`mapToGigaSOM`. Data is `unscatter`d after the computation. +$(TYPEDSIGNATURES) + +Overload of `assign` for simple data such as DataFrames and matrices. This +slices the data using `DistributedArrays`, sends them the workers, and runs +normal `assign`. Data is `unscatter`d after the computation. """ -function mapToGigaSOM(som::Som, data; knnTreeFun = BruteTree, metric = Euclidean()) +function assign(som::SOM, data; knnTreeFun = BruteTree, metric = Euclidean()) data = Matrix{Float64}(data) @@ -277,23 +257,19 @@ function mapToGigaSOM(som::Som, data; knnTreeFun = BruteTree, metric = Euclidean end dInfo = scatter_array(:GigaSOMmappingDataVar, data, workers()) - rInfo = mapToGigaSOM(som, dInfo, knnTreeFun = knnTreeFun, metric = metric) + rInfo = assign(som, dInfo, knnTreeFun = knnTreeFun, metric = metric) res = gather_array(rInfo) unscatter(dInfo) unscatter(rInfo) - return DataFrame(index = res) + return res end +export assign + """ - scaleEpochTime(iteration::Int64, epochs::Int64) +$(TYPEDSIGNATURES) Convert iteration ID and epoch number to relative time in training. """ -function scaleEpochTime(iteration::Int64, epochs::Int64) - # prevent division by zero on 1-epoch training - if epochs > 1 - epochs -= 1 - end - - return Float64(iteration - 1) / Float64(epochs) -end +scaled_epoch_time(iteration::Int64, epochs::Int64) = + Float64(iteration - 1) / Float64(max(epochs - 1, 1)) diff --git a/src/base/trainutils.jl b/src/som_utils.jl similarity index 68% rename from src/base/trainutils.jl rename to src/som_utils.jl index 589d7ecf..0b66da9b 100644 --- a/src/base/trainutils.jl +++ b/src/som_utils.jl @@ -1,8 +1,8 @@ """ - linearRadius(initRadius::Float64, iteration::Int64, decay::String, epochs::Int64) +$(TYPEDSIGNATURES) -Return a neighbourhood radius. Use as the `radiusFun` parameter for `trainGigaSOM`. +Return a neighbourhood radius. Use as the `radiusFun` parameter for `train`. # Arguments - `initRadius`: Initial Radius @@ -10,40 +10,41 @@ Return a neighbourhood radius. Use as the `radiusFun` parameter for `trainGigaSO - `iteration`: Training iteration - `epochs`: Total number of epochs """ -function linearRadius( +function radius_linear( initRadius::Float64, finalRadius::Float64, iteration::Int64, epochs::Int64, ) - scaledTime = scaleEpochTime(iteration, epochs) + scaledTime = scaled_epoch_time(iteration, epochs) return initRadius * (1 - scaledTime) + finalRadius * scaledTime end +export radius_linear """ - expRadius(steepness::Float64) +$(TYPEDSIGNATURES) -Return a function to be used as a `radiusFun` of `trainGigaSOM`, which causes +Return a function to be used as a `radiusFun` of `train`, which causes exponencial decay with the selected steepness. -Use: `trainGigaSOM(..., radiusFun = expRadius(0.5))` +Use: `train(..., radiusFun = radius_exp(0.5))` # Arguments - `steepness`: Steepness of exponential descent. Good values range from -100.0 (almost linear) to 100.0 (really quick decay). """ -function expRadius(steepness::Float64 = 0.0) +function radius_exp(steepness::Float64 = 0.0) return (initRadius::Float64, finalRadius::Float64, iteration::Int64, epochs::Int64) -> begin - scaledTime = scaleEpochTime(iteration, epochs) + scaledTime = scaled_epoch_time(iteration, epochs) if steepness < -100.0 # prevent floating point underflows - error("Sanity check: steepness too low, use linearRadius instead.") + error("Sanity check: steepness too low, use radius_linear instead.") end # steepness is simulated by moving both points closer to zero @@ -62,9 +63,10 @@ function expRadius(steepness::Float64 = 0.0) end end +export radius_exp """ - gridRectangular(xdim, ydim) +$(TYPEDSIGNATURES) Create coordinates of all neurons on a rectangular SOM. @@ -79,7 +81,7 @@ The first neuron sits at (0,0). - `xdim`: number of neurons in x-direction - `ydim`: number of neurons in y-direction """ -function gridRectangular(xdim, ydim) +function grid_rectangular(xdim, ydim) grid = zeros(Float64, (xdim * ydim, 2)) for ix = 1:xdim @@ -91,57 +93,54 @@ function gridRectangular(xdim, ydim) return grid end +export grid_rectangular """ - gaussianKernel(x, r::Float64) +$(TYPEDSIGNATURES) Return the value of normal distribution PDF (σ=`r`, μ=0) at `x` """ -function gaussianKernel(x, r::Float64) +kernel_gaussian(x, r::Float64) = Distributions.pdf.(Distributions.Normal(0.0, r), x) - return Distributions.pdf.(Distributions.Normal(0.0, r), x) -end +export kernel_gaussian -function bubbleKernelSqScalar(x::Float64, r::Float64) - if x >= r - return 0 - else - return sqrt(1 - x / r) - end -end +bubble_kernel_squared_scalar(x::Float64, r::Float64) = x >= r ? 0 : sqrt(1 - x / r) """ - bubbleKernel(x, r::Float64) +$(TYPEDSIGNATURES) Return a "bubble" (spherical) distribution kernel. """ -function bubbleKernel(x, r::Float64) - return bubbleKernelSqScalar.(x .^ 2, r^2) -end +kernel_bubble(x, r::Float64) = bubble_kernel_squared_scalar.(x .^ 2, r^2) + +export kernel_bubble """ - thresholdKernel(x, r::Float64) +$(TYPEDSIGNATURES) + kernel_threshold(x, r::Float64) Simple FlowSOM-like hard-threshold kernel """ -function thresholdKernel(x, r::Float64, maxRatio = 4 / 5, zero = 1e-6) +function kernel_threshold(x, r::Float64, maxRatio = 4 / 5, zero = 1e-6) if r >= maxRatio * maximum(x) #prevent smoothing everything to a single point r = maxRatio * maximum(x) end return zero .+ (x .<= r) end +export kernel_threshold + """ - distMatrix(metric=Chebyshev()) +$(TYPEDSIGNATURES) Return a function that uses the `metric` (compatible with metrics from package `Distances`) calculates distance matrixes from normal row-wise data matrices, using the `metric`. -Use as a parameter of `trainGigaSOM`. +Use as a parameter of `train`. """ -function distMatrix(metric = Chebyshev()) - return (grid::Matrix{Float64}) -> begin +distance_matrix(metric = Chebyshev()) = + (grid::Matrix{Float64}) -> begin n = size(grid, 1) dm = zeros(Float64, n, n) @@ -153,4 +152,5 @@ function distMatrix(metric = Chebyshev()) return dm::Matrix{Float64} end -end + +export distance_matrix diff --git a/src/io/splitting.jl b/src/split.jl similarity index 76% rename from src/io/splitting.jl rename to src/split.jl index 98dbdc22..b7f929cc 100644 --- a/src/io/splitting.jl +++ b/src/split.jl @@ -1,5 +1,5 @@ """ - slicesof(lengths::Vector{Int}, slices::Int)::Vector{Tuple{Int,Int,Int,Int}} +$(TYPEDSIGNATURES) Given a list of `lengths` of input arrays, compute a slicing into a specified amount of equally-sized `slices`. @@ -46,7 +46,7 @@ function slicesof(lengths::Vector{Int}, slices::Int)::Vector{Tuple{Int,Int,Int,I end """ - vcollectSlice(loadMtx, (startFile, startOff, finalFile, finalOff)::Tuple{Int,Int,Int,Int})::Matrix +$(TYPEDSIGNATURES) Given a method to obtain matrix content (`loadMtx`), reconstruct a slice from the information generated by `slicesof`. @@ -58,24 +58,11 @@ therefore named _v_collect (the slicing and concatenation is _v_ertical). The actual data content and loading method is abstracted out -- function `loadMtx` gets the index of the input part that it is required to fetch (e.g. index of one FCS file), and is expected to return that input part as a whole -matrix. `vcollectSlice` correctly calls this function as required and extracts +matrix. `vcollect_slice` correctly calls this function as required and extracts relevant portions of the matrices, so that at the end the whole slice can be pasted together. - -Example: - - # get a list of files - filenames=["a.fcs", "b.fcs"] - # get descriptions of 5 equally sized parts of the data - slices = slicesof(loadFCSSizes(filenames), 5) - - # reconstruct first 3 columns of the first slice - mySlice = vcollectSlice( - i -> last(loadFCS(slices[i]))[:,1:3], - slices[1]) - # (note: function loadFCS returns 4 items, the matrix is the last one) """ -function vcollectSlice( +function vcollect_slice( loadMtx, (startFile, startOff, finalFile, finalOff)::Tuple{Int,Int,Int,Int}, )::Matrix @@ -92,11 +79,11 @@ function vcollectSlice( end """ - collectSlice(loadVec, (startFile, startOff, finalFile, finalOff)::Tuple{Int,Int,Int,Int})::Vector +$(TYPEDSIGNATURES) -Alternative of `vcollectSlice` for 1D vectors. +Alternative of `vcollect_slice` for 1D vectors. """ -function collectSlice( +function collect_slice( loadVec, (startFile, startOff, finalFile, finalOff)::Tuple{Int,Int,Int,Int}, )::Vector diff --git a/src/base/structs.jl b/src/structs.jl similarity index 50% rename from src/base/structs.jl rename to src/structs.jl index b9689698..026a8e33 100644 --- a/src/base/structs.jl +++ b/src/structs.jl @@ -1,25 +1,19 @@ """ - Som +$(TYPEDEF) Structure to hold all data of a trained SOM. # Fields: -- `codes::Array{Float64,2}`: 2D-array of codebook vectors. One vector per row -- `xdim::Int`: number of neurons in x-direction -- `ydim::Int`: number of neurons in y-direction -- `numCodes::Int`: total number of neurons -- `grid::Array{Float64,2}`: 2D-array of coordinates of neurons on the map - (2 columns (x,y)] for rectangular and hexagonal maps - 3 columns (x,y,z) for spherical maps) +$(TYPEDFIELDS) """ -mutable struct Som +mutable struct SOM codes::Matrix{Float64} xdim::Int ydim::Int numCodes::Int grid::Matrix{Float64} - Som(; + SOM(; codes::Matrix{Float64}, xdim::Int, ydim::Int, @@ -28,7 +22,9 @@ mutable struct Som ) = new(codes, xdim, ydim, numCodes, grid) end -Base.copy(som::Som) = Som( +export SOM + +Base.copy(som::SOM) = SOM( codes = som.codes, xdim = som.xdim, ydim = som.ydim, diff --git a/test/aqua.jl b/test/aqua.jl new file mode 100644 index 00000000..89c53116 --- /dev/null +++ b/test/aqua.jl @@ -0,0 +1,4 @@ + +@testset "Aqua.jl" begin + Aqua.test_all(GigaSOM;) +end diff --git a/test/coverage/coverage-summary.jl b/test/coverage/coverage-summary.jl deleted file mode 100644 index a0fa0459..00000000 --- a/test/coverage/coverage-summary.jl +++ /dev/null @@ -1,12 +0,0 @@ - -using Coverage - -cd(joinpath(@__DIR__, "..", "..")) do - processed = process_folder() - covered_lines, total_lines = get_summary(processed) - percentage = covered_lines / total_lines * 100 - println("($(percentage)%) covered") -end - -# submit the report to codecov.io -Codecov.submit_local(Codecov.process_folder(), pwd(), slug = "LCSB-BioCore/GigaSOM.jl") diff --git a/test/runtests.jl b/test/runtests.jl index 8d8414d5..bd7596ee 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,36 +1,29 @@ -using GigaSOM, DataFrames, XLSX, CSV, Test, Random, Distributed, DistributedData -using FileIO, DataFrames, Distances -using JSON, SHA -import LinearAlgebra -owd = pwd() +using Test +using Aqua +import GigaSOM -""" -Check if the `pwd()` is the `/test` directory, and if not it changes to it. -""" -function checkDir() - files = readdir() - if !in("runtests.jl", files) - cd(dirname(dirname(pathof(GigaSOM)))) - end +# helper functions for running tests en masse +print_timing(fn, t) = @info "$(fn) done in $(round(t; digits = 2))s" + +function run_test_file(path...) + fn = joinpath(path...) + t = @elapsed include(fn) + print_timing(fn, t) end -checkDir() +function run_doc_examples() + for ex in filter(endswith(".jl"), readdir("../docs/src/tutorials", join = true)) + @testset "docs/$(basename(ex))" begin + run_test_file(ex) + end + end +end @testset "GigaSOM test suite" begin - include("testDataOps.jl") - include("testTrainutils.jl") - include("testSplitting.jl") - include("testInput.jl") - - #this loads the PBMC dataset required for the batch/parallel tests - include("testLoadPBMC8.jl") - include("testBatch.jl") - include("testParallel.jl") + @testset "Documentation tests" begin + run_doc_examples() + end - #misc tests that require some of the above data too - include("testInputCSV.jl") - include("testFileSplitting.jl") + run_test_file("aqua.jl") end - -cd(owd) diff --git a/test/testBatch.jl b/test/testBatch.jl index 2f8f3587..ad556bba 100644 --- a/test/testBatch.jl +++ b/test/testBatch.jl @@ -1,11 +1,11 @@ @testset "Single-CPU batch processing" begin - som = initGigaSOM(pbmc8_data, 10, 10, seed = 1234) + som = init(pbmc8_data, 10, 10, seed = 1234) #check whether the distributed version works the same save_at(1, :test, pbmc8_data) - som2 = initGigaSOM(Dinfo(:test, [1]), 10, 10, seed = 1234) + som2 = init(Dinfo(:test, [1]), 10, 10, seed = 1234) @test som.codes == som2.codes remove_from(1, :test) @@ -16,11 +16,11 @@ @test som.numCodes == 100 end - som = trainGigaSOM(som, pbmc8_data, epochs = 1) + som = train(som, pbmc8_data, epochs = 1) - winners = mapToGigaSOM(som, pbmc8_data) + winners = assign(som, pbmc8_data) - embed = embedGigaSOM(som, pbmc8_data, k = 10, smooth = 0.1, adjust = 2.3, m = 4.5) + e = embed(som, pbmc8_data, k = 10, smooth = 0.1, adjust = 2.3, m = 4.5) @testset "Check results" begin codes = som.codes @@ -28,9 +28,9 @@ dfCodes = DataFrame(codes, :auto) rename!(dfCodes, Symbol.(antigens)) - dfEmbed = DataFrame(embed, :auto) + dfEmbed = DataFrame(e, :auto) CSV.write(genDataPath * "/batchDfCodes.csv", dfCodes) - CSV.write(genDataPath * "/batchWinners.csv", winners) + CSV.write(genDataPath * "/batchWinners.csv", DataFrame(index = winners)) CSV.write(genDataPath * "/batchEmbedded.csv", dfEmbed) #load the ref data diff --git a/test/testFileSplitting.jl b/test/testFileSplitting.jl index b2e089be..24a88ec2 100644 --- a/test/testFileSplitting.jl +++ b/test/testFileSplitting.jl @@ -1,3 +1,5 @@ +import GigaSOM: read_fcs_sizes, fcs_filevector_distribute + @testset "File splitting" begin W = addprocs(8) @@ -7,7 +9,7 @@ # try load it by parts of different size and scrutinize the parts for splitSize = 1:8 - di = loadFCSSet(:test, md[:, :file_name], W[1:splitSize]) + di = load_fcs_distributed(:test, md[:, :file_name], W[1:splitSize]) dselect(di, fcsAntigens, antigens) cols = Vector(1:length(antigens)) dtransform_asinh(di, cols, 5) @@ -18,8 +20,8 @@ @test sum(splits) == size(pbmc8_data, 1) @test minimum(splits) + 1 >= maximum(splits) - sizes = loadFCSSizes(md[:, :file_name]) - dis = distributeFCSFileVector(:testF, md[:, :file_name], W[1:splitSize]) + sizes = read_fcs_sizes(md[:, :file_name]) + dis = fcs_filevector_distribute(:testF, md[:, :file_name], W[1:splitSize]) @test dcount(length(sizes), dis) == sizes @test dcount_buckets(length(sizes), dis, length(sizes), dis) == Matrix{Int}(LinearAlgebra.Diagonal(sizes)) diff --git a/test/testInput.jl b/test/testInput.jl index 4dc1a667..e9d1d3bb 100644 --- a/test/testInput.jl +++ b/test/testInput.jl @@ -2,7 +2,7 @@ checkDir() @testset "File loading with compensation" begin - _, data = loadFCS("test-compensation.fcs") + _, data = load_fcs("test-compensation.fcs") @test isapprox( data[:, 7:18], [ diff --git a/test/testInputCSV.jl b/test/testInputCSV.jl index 7477da63..72e607e2 100644 --- a/test/testInputCSV.jl +++ b/test/testInputCSV.jl @@ -1,22 +1,24 @@ +import GigaSOM: read_csv_size, read_csv_sizes + @testset "CSV loading" begin files = [refDataPath * "/refBatchDfCodes.csv", refDataPath * "/refParallelDfCodes.csv"] - rows, cols = getCSVSize(files[1], header = true) + rows, cols = read_csv_size(files[1], header = true) @test rows == 10 @test cols == 10 - data1 = loadCSV(files[1], header = true) - data2 = loadCSV(files[2], header = true) + data1 = load_csv(files[1], header = true) + data2 = load_csv(files[2], header = true) @test typeof(data1) == Matrix{Float64} @test size(data1) == (10, 10) - sizes = loadCSVSizes(files, header = true) + sizes = read_csv_sizes(files, header = true) @test sizes == [10, 10] W = addprocs(3) @everywhere using GigaSOM - di = loadCSVSet(:csvTest, files, W, header = true) + di = load_csv_distributed(:csvTest, files, W, header = true) @test gather_array(di) == vcat(data1, data2) sizes = dmapreduce(di, size, vcat) dims = map(last, sizes) diff --git a/test/testLoadPBMC8.jl b/test/testLoadPBMC8.jl index e1328b6a..af9b5f49 100644 --- a/test/testLoadPBMC8.jl +++ b/test/testLoadPBMC8.jl @@ -1,4 +1,6 @@ +import GigaSOM: read_fcs_header + # This loads the PBMC8 dataset that is used for later tests checkDir() @@ -42,7 +44,7 @@ end fileNames = readdir() csDict = Dict{String,Any}() for f in fileNames - if f[end-3:end] == ".fcs" || f[end-4:end] == ".xlsx" + if f[(end-3):end] == ".fcs" || f[(end-4):end] == ".xlsx" cs = bytes2hex(sha256(f)) csDict[f] = cs end @@ -56,13 +58,13 @@ end md = DataFrame(XLSX.readtable("PBMC8_metadata.xlsx", "Sheet1", infer_eltypes = true)) panel = DataFrame(XLSX.readtable("PBMC8_panel.xlsx", "Sheet1", infer_eltypes = true)) -antigens = panel[panel[:, :Lineage].==1, :Antigen] -_, fcsParams = loadFCSHeader(md[1, :file_name]) -_, fcsAntigens = getMarkerNames(getMetaData(fcsParams)) -cleanNames!(antigens) -cleanNames!(fcsAntigens) +antigens = panel[panel[:, :Lineage] .== 1, :Antigen] +_, fcsParams = read_fcs_header(md[1, :file_name]) +_, fcsAntigens = fcs_metadata_marker_names(fcs_column_metadata(fcsParams)) +clean_names!(antigens) +clean_names!(fcsAntigens) -di = loadFCSSet(:fcsData, md[:, :file_name], [myid()]) +di = load_fcs_distributed(:fcsData, md[:, :file_name], [myid()]) #prepare the data a bit dselect(di, fcsAntigens, antigens) @@ -74,11 +76,11 @@ pbmc8_data = gather_array(di) unscatter(di) @testset "load-time columns normalization" begin - di = loadFCSSet( + di = load_fcs_distributed( :fcsData, md[:, :file_name], [myid()], - postLoad = selectFCSColumns(antigens), + postLoad = select_fcs_columns(antigens), ) dtransform_asinh(di, cols, 5) dscale(di, cols) diff --git a/test/testParallel.jl b/test/testParallel.jl index 0fc3153e..9188dc0d 100644 --- a/test/testParallel.jl +++ b/test/testParallel.jl @@ -4,7 +4,7 @@ W = addprocs(2) @everywhere using GigaSOM - som = initGigaSOM(pbmc8_data, 10, 10, seed = 1234) + som = init(pbmc8_data, 10, 10, seed = 1234) @testset "Check SOM dimensions" begin @test size(som.codes) == (100, 10) @@ -13,11 +13,11 @@ @test som.numCodes == 100 end - som = trainGigaSOM(som, pbmc8_data, epochs = 2, rStart = 6.0) + som = train(som, pbmc8_data, epochs = 2, rStart = 6.0) - winners = mapToGigaSOM(som, pbmc8_data) + winners = assign(som, pbmc8_data) - embed = embedGigaSOM(som, pbmc8_data) + e = embed(som, pbmc8_data) @testset "Check results" begin codes = som.codes @@ -25,9 +25,9 @@ dfCodes = DataFrame(codes, :auto) rename!(dfCodes, Symbol.(antigens)) - dfEmbed = DataFrame(embed, :auto) + dfEmbed = DataFrame(e, :auto) CSV.write(genDataPath * "/parallelDfCodes.csv", dfCodes) - CSV.write(genDataPath * "/parallelWinners.csv", winners) + CSV.write(genDataPath * "/parallelWinners.csv", DataFrame(index = winners)) CSV.write(genDataPath * "/parallelEmbedded.csv", dfEmbed) # load the ref data diff --git a/test/testSplitting.jl b/test/testSplitting.jl index 50ecf5e4..d1d60aee 100644 --- a/test/testSplitting.jl +++ b/test/testSplitting.jl @@ -1,4 +1,6 @@ +import GigaSOM: slicesof, vcollect_slice + @testset "Dataset splitting helpers" begin @testset "slice computation" begin @@ -34,8 +36,8 @@ @testset "slice collection" begin s = slicesof([4, 4], 3) - @test vcollectSlice(i -> repeat([i], 4), s[1])[:, 1] == [1, 1, 1] - @test vcollectSlice(i -> repeat([i], 4), s[2])[:, 1] == [1, 2, 2] - @test vcollectSlice(i -> repeat([i], 4), s[3])[:, 1] == [2, 2] + @test vcollect_slice(i -> repeat([i], 4), s[1])[:, 1] == [1, 1, 1] + @test vcollect_slice(i -> repeat([i], 4), s[2])[:, 1] == [1, 2, 2] + @test vcollect_slice(i -> repeat([i], 4), s[3])[:, 1] == [2, 2] end end diff --git a/test/testTrainutils.jl b/test/testTrainutils.jl index 57b3d58e..470b0c2c 100644 --- a/test/testTrainutils.jl +++ b/test/testTrainutils.jl @@ -1,37 +1,37 @@ @testset "SOM training helper functions" begin - @testset "gridRectangular" begin - grid = GigaSOM.gridRectangular(5, 5) + @testset "grid_rectangular" begin + grid = GigaSOM.grid_rectangular(5, 5) @test size(grid) == (25, 2) end @testset "Kernels" begin @test isapprox( - gaussianKernel(Vector{Float64}(1:7), 2.0), + kernel_gaussian(Vector{Float64}(1:7), 2.0), [0.1760326, 0.1209853, 0.0647587, 0.0269954, 0.0087641, 0.0022159, 0.0004363], atol = 1e-4, ) @test isapprox( - bubbleKernel(Vector{Float64}(1:6), 5.0), + kernel_bubble(Vector{Float64}(1:6), 5.0), [0.979796, 0.916515, 0.8, 0.6, 0, 0], atol = 0.001, ) @test isapprox( - thresholdKernel(Vector{Float64}(1:10), 5.0), + kernel_threshold(Vector{Float64}(1:10), 5.0), [1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0], atol = 1e-3, ) @test isapprox( - thresholdKernel(Vector{Float64}(1:10), 10.1), + kernel_threshold(Vector{Float64}(1:10), 10.1), [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0], atol = 1e-3, ) end @testset "Radii-generating functions" begin - radii1 = expRadius().(5.0, 0.5, Vector(1:10), 10) - radii2 = expRadius(-10.0).(10.0, 0.1, Vector(1:20), 20) - radii3 = linearRadius.(50.0, 0.001, Vector(1:30), 30) + radii1 = radius_exp().(5.0, 0.5, Vector(1:10), 10) + radii2 = radius_exp(-10.0).(10.0, 0.1, Vector(1:20), 20) + radii3 = radius_linear.(50.0, 0.001, Vector(1:30), 30) @test isapprox(radii1[1], 5.0) @test isapprox(radii1[10], 0.5) @test isapprox(radii2[1], 10.0) @@ -43,9 +43,9 @@ @test all(isapprox.(radii3[1:29] .- radii3[2:30], radii3[1] - radii3[2])) end - @testset "distMatrix" begin - g = GigaSOM.gridRectangular(2, 2) - dm = GigaSOM.distMatrix(Euclidean())(g) + @testset "distance_matrix" begin + g = GigaSOM.grid_rectangular(2, 2) + dm = GigaSOM.distance_matrix(Euclidean())(g) @test size(dm) == (4, 4) @test all([dm[i, i] == 0 for i = 1:4]) end