Initial commit

---------

Co-authored-by: Jackmin801 <[email protected]>
Co-authored-by: Jackmin801 <[email protected]>

.dockerignore

@@ -0,0 +1,94 @@
+# Training
+
+*/wandb/*
+*/data/*
+*/outputs/*
+
+# Git
+.git
+.gitignore
+
+# Docker
+docker-compose.yml
+.docker
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*/__pycache__/
+*/*/__pycache__/
+*/*/*/__pycache__/
+*.py[cod]
+*/*.py[cod]
+*/*/*.py[cod]
+*/*/*/*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Virtual environment
+.env/
+.venv/
+venv/
+
+# PyCharm
+.idea
+
+# Python mode for VIM
+.ropeproject
+*/.ropeproject
+*/*/.ropeproject
+*/*/*/.ropeproject
+
+# Vim swap files
+*.swp
+*/*.swp
+*/*/*.swp
+*/*/*/*.swp

.github/workflows/check-style.yml

@@ -0,0 +1,25 @@
+name: Check style
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    # This will trigger the workflow for pull requests to any branch
+    types: [opened, synchronize, reopened]
+
+jobs:
+  ruff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: chartboost/ruff-action@v1
+
+  codespell:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: codespell-project/actions-codespell@v1
+        with:
+          only_warn: 1
+          ignore_words_list: ibrary,nd

.github/workflows/push-docker-image.yml

@@ -0,0 +1,61 @@
+name: Push to Docker Hub
+
+on:
+  push:
+    branches: [ main ]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Remove unnecessary packages
+        run: |
+          echo "=== Before pruning ==="
+          df -h
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          echo "=== After pruning ==="
+          df -h
+
+      # Link to discussion: https://github.com/orgs/community/discussions/25678
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: Docker meta
+        id: meta
+        uses: crazy-max/ghaction-docker-meta@v2
+        with:
+          images: |
+            primeintellect/open_diloco
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha,prefix=commit-
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

.gitignore

@@ -0,0 +1,84 @@
+# node and NPM
+npm-debug.log
+node_modules
+
+# swap files
+*~
+*.swp
+
+examples/data/*
+examples/runs/*
+examples/.ipynb_checkpoints/*
+
+env.sh
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg/
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+.idea
+.vscode
+.ipynb_checkpoints
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+# Sphinx documentation
+docs/_build/
+docs/tmp*
+
+# OS X garbage
+.DS_Store
+
+# Debian things
+debian/reproducible-experiment-platform
+debian/files
+*.substvars
+*.debhelper.log
+
+# protobuf stuff
+hivemind/proto/*_pb2*
+
+# libp2p-daemon binary
+hivemind/hivemind_cli/p2pd

.gitmodules

@@ -0,0 +1,7 @@
+[submodule "hivemind_source"]
+	path = hivemind_source
+	url = https://github.com/PrimeIntellect-ai/hivemind.git
+	branch = feat-add-downloading-time
+[submodule "pydantic_config"]
+	path = pydantic_config
+	url = https://github.com/samsja/pydantic_config.git

.pre-commit-config.yaml

@@ -0,0 +1,10 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.4.7
+  hooks:
+    # Run the linter.
+    - id: ruff
+      args: [ --fix ]
+    # Run the formatter.
+    - id: ruff-format

CONTRIBUTING.md

@@ -0,0 +1,36 @@
+# Development workflow
+
+This is the develpment workflow of prime intellect to build upon hivemind
+
+## Install dependencies
+
+Install hivemind  
+
+```bash
+cd hivemind_source
+pip install .
+cp build/lib/hivemind/proto/* hivemind/proto/.
+pip install -e ".[all]"```
+```
+
+## Pre-commit hook
+
+Install the pre commit hook to keep black and isort updated on each commit:
+
+```
+pre-commit install
+```
+
+## Testing
+To run the tests:
+
+```
+python -m pytest tests
+```
+
+Be sure to actually use python -m otherwise path won't be appended correctly
+
+# Development flags
+Add the `PRIME_INTELLECT_DEV` environment variable to your *.bashrc* or *.zshrc* so that development features are enabled.
+e.g.
+- torch compile error will crash the script instead of silently failing

Dockerfile

@@ -0,0 +1,46 @@
+FROM pytorch/pytorch:2.3.1-cuda12.1-cudnn8-devel
+LABEL maintainer="prime intellect"
+LABEL repository="open_diloco"
+
+# Set en_US.UTF-8 locale by default
+RUN echo "LC_ALL=en_US.UTF-8" >> /etc/environment
+
+# Set CUDA_HOME and update PATH
+ENV CUDA_HOME=/usr/local/cuda
+ENV PATH=$PATH:/usr/local/cuda/bin
+
+# Install packages
+RUN apt-get update && apt-get install -y --no-install-recommends --force-yes \
+  build-essential \
+  curl \
+  wget \
+  git \
+  vim \
+  htop \
+  nvtop \
+  iperf \
+  tmux \
+  openssh-server \   
+  git-lfs \   
+  && apt-get clean autoclean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Install Git LFS
+RUN git-lfs install
+
+# Install Rust
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+RUN echo "export PATH=\"/opt/conda/bin:/root/.cargo/bin:\$PATH\"" >> /root/.bashrc
+
+# Install Python dependencies (The gradual copies help with caching)
+WORKDIR open_diloco
+RUN pip install --pre torchdata --index-url https://download.pytorch.org/whl/nightly/cpu
+COPY hivemind_source hivemind_source
+RUN pip install --no-cache-dir ./hivemind_source
+COPY requirements.txt requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+COPY requirements-dev.txt requirements-dev.txt
+RUN pip install --no-cache-dir -r requirements-dev.txt
+COPY . .
+RUN pip install .
+RUN rm -rf ~/.cache/pip