Skip to content

Commit

Permalink
Init OSS repo
Browse files Browse the repository at this point in the history
  • Loading branch information
juberti committed May 29, 2024
0 parents commit 661bc9a
Show file tree
Hide file tree
Showing 103 changed files with 15,502 additions and 0 deletions.
40 changes: 40 additions & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Run tests

on:
push:
branches: [main, prod]
pull_request:
branches: [main, prod]

jobs:
test:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
with:
lfs: true

- uses: extractions/setup-just@v1

- name: Set up Python 3.11
uses: actions/setup-python@v2
with:
python-version: 3.11

- name: Install dependencies
run: |
just install
- name: Run checks
run: |
just check
- name: Run tests
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
run: |
just test
191 changes: 191 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
.vscode/
.venv/
.hermit/
runs/
outputs/
wandb/

artifacts/

output_audio_processor/
output_tokenizer/

*.csv
*.json
epd_eval/
.git/
env_vars.sh
sync_watch
clearml.conf
aml.md
.DS_Store
*.safetensors
*.pt

# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# C extensions
*.so

# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST

# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
.pybuilder/
target/

# Jupyter Notebook
.ipynb_checkpoints

# IPython
profile_default/
ipython_config.py

# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version

# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock

# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock

# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
.pdm.toml

# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/

# Celery stuff
celerybeat-schedule
celerybeat.pid

# SageMath parsed files
*.sage.py

# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/

# Spyder project settings
.spyderproject
.spyproject

# Rope project settings
.ropeproject

# mkdocs documentation
/site

# mypy
.mypy_cache/
.dmypy.json
dmypy.json

# Pyre type checker
.pyre/

# pytype static type analyzer
.pytype/

# Cython debug symbols
cython_debug/

# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/
.vscode/

.neptune/
mds_output/
mlruns/
output/

67 changes: 67 additions & 0 deletions Justfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
export WANDB_PROJECT:="ultravox"
export WANDB_LOG_MODEL:="checkpoint"
export PROJECT_DIR:="ultravox"
export VENV_NAME:="venv"
export MCLOUD_CLUSTER:="r7z22"
export MCLOUD_INSTANCE:="oci.bm.gpu.b4.8"

default: format check test

create-venv:
pip install --upgrade virtualenv # older virtualenv had some issues in Debian
python -m venv ${VENV_NAME}
just install

install:
# Install torch 2.2.1 if needed, not present in requirements.txt
just python -c \"import torch\" 2>/dev/null || just pip install torch==2.2.1
just pip install -r requirements.txt
just pip install -r requirements-dev.txt
just python -m pip install types-requests
format:
. ./activate ${VENV_NAME} && autoflake ${PROJECT_DIR} --remove-all-unused-imports --quiet --in-place -r --exclude third_party --exclude ultravox/model/gazelle
. ./activate ${VENV_NAME} && isort ${PROJECT_DIR} --force-single-line-imports
. ./activate ${VENV_NAME} && black ${PROJECT_DIR}
check:
. ./activate ${VENV_NAME} && black ${PROJECT_DIR} --check
. ./activate ${VENV_NAME} && isort ${PROJECT_DIR} --check --force-single-line-imports
. ./activate ${VENV_NAME} && autoflake ${PROJECT_DIR} --check --quiet --remove-all-unused-imports -r --exclude third_party --exclude ultravox/model/gazelle
. ./activate ${VENV_NAME} && mypy ${PROJECT_DIR}
test *ARGS=".":
. ./activate ${VENV_NAME} && cd ${PROJECT_DIR} && pytest --ignore third_party {{ARGS}}
@python *FLAGS:
. ./activate ${VENV_NAME} && python {{FLAGS}}
@pip *FLAGS:
. ./activate ${VENV_NAME} && pip {{FLAGS}}
train *FLAGS:
just python -m ultravox.training.train {{FLAGS}}
train_asr *FLAGS:
just train --config_path ultravox/training/configs/asr_tinyllama.yaml {{FLAGS}}
browse *FLAGS:
just python -m ultravox.tools.data_tool {{FLAGS}}
infer *FLAGS:
just python -m ultravox.tools.infer_tool {{FLAGS}}
eval *FLAGS:
just python -m ultravox.tools.eval_tool {{FLAGS}}
mds *FLAGS:
just python -m ultravox.tools.mds_tool {{FLAGS}}
gradio *FLAGS:
just python -m ultravox.tools.gradio_demo {{FLAGS}}
run *FLAGS:
mcli run -f mcloud.yaml --follow {{FLAGS}}
mcloud *FLAGS:
mcli interactive {{FLAGS}} --cluster ${MCLOUD_CLUSTER} --instance ${MCLOUD_INSTANCE} --name `whoami` --command "bash -c \"$(cat setup.sh)\""
Loading

0 comments on commit 661bc9a

Please sign in to comment.