Spaces:
Running
Running
AndrΓ© Oliveira commited on
Commit Β·
6472a71
1
Parent(s): 7c32592
v0
Browse files- .all-installed-packages.txt +110 -0
- .gitignore +229 -0
- LICENSE +209 -0
- README.md +151 -10
- app.py +0 -0
- assets/guitar.png +0 -0
- assets/harp.png +0 -0
- assets/music-box.png +0 -0
- assets/ocarina.png +0 -0
- assets/piano.png +0 -0
- assets/xylophone.png +0 -0
- draw/__init__.py +0 -0
- draw/strokes.py +278 -0
- draw/vision.py +351 -0
- requirements.txt +33 -0
- synths/__init__.py +0 -0
- synths/guitar.py +393 -0
- synths/harp.py +381 -0
- synths/musicbox.py +320 -0
- synths/ocarina.py +197 -0
- synths/piano.py +234 -0
- synths/voice.py +107 -0
- synths/xylophone.py +298 -0
- tests/test_audio_pipeline.py +106 -0
- tests/test_llm.py +163 -0
- tests/test_voice.py +102 -0
- train/generate_dataset.py +599 -0
- train/generate_dataset_requirements.txt +4 -0
- train/train_lullaby.ipynb +0 -0
- utils/__init__.py +0 -0
- utils/safety.py +95 -0
- utils/trace.py +255 -0
.all-installed-packages.txt
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
addict==2.4.0
|
| 2 |
+
annotated-doc==0.0.4
|
| 3 |
+
annotated-types==0.7.0
|
| 4 |
+
anyio==4.13.0
|
| 5 |
+
attrs==26.1.0
|
| 6 |
+
babel==2.18.0
|
| 7 |
+
blis==1.3.3
|
| 8 |
+
brotli==1.2.0
|
| 9 |
+
catalogue==2.0.10
|
| 10 |
+
certifi==2026.4.22
|
| 11 |
+
cffi==2.0.0
|
| 12 |
+
charset-normalizer==3.4.7
|
| 13 |
+
click==8.3.3
|
| 14 |
+
cloudpathlib==0.24.0
|
| 15 |
+
confection==1.3.3
|
| 16 |
+
csvw==4.0.0
|
| 17 |
+
curated-tokenizers==0.0.9
|
| 18 |
+
curated-transformers==0.1.1
|
| 19 |
+
cymem==2.0.13
|
| 20 |
+
diskcache==5.6.3
|
| 21 |
+
dlinfo==2.0.0
|
| 22 |
+
docopt==0.6.2
|
| 23 |
+
en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl#sha256=1932429db727d4bff3deed6b34cfc05df17794f4a52eeb26cf8928f7c1a0fb85
|
| 24 |
+
espeakng-loader==0.2.4
|
| 25 |
+
fastapi==0.136.1
|
| 26 |
+
filelock==3.29.0
|
| 27 |
+
fsspec==2026.4.0
|
| 28 |
+
gradio==6.14.0
|
| 29 |
+
gradio_client==2.5.0
|
| 30 |
+
groovy==0.1.2
|
| 31 |
+
h11==0.16.0
|
| 32 |
+
hf-gradio==0.4.1
|
| 33 |
+
hf-xet==1.5.0
|
| 34 |
+
httpcore==1.0.9
|
| 35 |
+
httpx==0.28.1
|
| 36 |
+
huggingface_hub==1.15.0
|
| 37 |
+
idna==3.15
|
| 38 |
+
isodate==0.7.2
|
| 39 |
+
Jinja2==3.1.6
|
| 40 |
+
joblib==1.5.3
|
| 41 |
+
jsonschema==4.26.0
|
| 42 |
+
jsonschema-specifications==2025.9.1
|
| 43 |
+
kokoro==0.9.4
|
| 44 |
+
language-tags==1.3.1
|
| 45 |
+
llama_cpp_python==0.3.23
|
| 46 |
+
loguru==0.7.3
|
| 47 |
+
markdown-it-py==4.2.0
|
| 48 |
+
MarkupSafe==3.0.3
|
| 49 |
+
mdurl==0.1.2
|
| 50 |
+
misaki==0.9.4
|
| 51 |
+
mpmath==1.3.0
|
| 52 |
+
murmurhash==1.0.15
|
| 53 |
+
networkx==3.6.1
|
| 54 |
+
num2words==0.5.14
|
| 55 |
+
numpy==2.4.5
|
| 56 |
+
orjson==3.11.9
|
| 57 |
+
packaging==26.2
|
| 58 |
+
pandas==3.0.3
|
| 59 |
+
phonemizer-fork==3.3.2
|
| 60 |
+
pillow==12.2.0
|
| 61 |
+
preshed==3.0.13
|
| 62 |
+
pycparser==3.0
|
| 63 |
+
pydantic==2.13.4
|
| 64 |
+
pydantic_core==2.46.4
|
| 65 |
+
pydub==0.25.1
|
| 66 |
+
Pygments==2.20.0
|
| 67 |
+
pyparsing==3.3.2
|
| 68 |
+
python-dateutil==2.9.0.post0
|
| 69 |
+
python-multipart==0.0.28
|
| 70 |
+
pytz==2026.2
|
| 71 |
+
PyYAML==6.0.3
|
| 72 |
+
rdflib==7.6.0
|
| 73 |
+
referencing==0.37.0
|
| 74 |
+
regex==2026.5.9
|
| 75 |
+
requests==2.34.2
|
| 76 |
+
rfc3986==1.5.0
|
| 77 |
+
rich==15.0.0
|
| 78 |
+
rpds-py==0.30.0
|
| 79 |
+
safehttpx==0.1.7
|
| 80 |
+
safetensors==0.7.0
|
| 81 |
+
scipy==1.17.1
|
| 82 |
+
segments==2.4.0
|
| 83 |
+
semantic-version==2.10.0
|
| 84 |
+
shellingham==1.5.4
|
| 85 |
+
six==1.17.0
|
| 86 |
+
smart_open==7.6.1
|
| 87 |
+
soundfile==0.13.1
|
| 88 |
+
spacy==3.8.14
|
| 89 |
+
spacy-curated-transformers==0.3.1
|
| 90 |
+
spacy-legacy==3.0.12
|
| 91 |
+
spacy-loggers==1.0.5
|
| 92 |
+
srsly==2.5.3
|
| 93 |
+
starlette==1.0.0
|
| 94 |
+
sympy==1.14.0
|
| 95 |
+
termcolor==3.3.0
|
| 96 |
+
thinc==8.3.13
|
| 97 |
+
tokenizers==0.22.2
|
| 98 |
+
tomlkit==0.14.0
|
| 99 |
+
torch==2.11.0
|
| 100 |
+
tqdm==4.67.3
|
| 101 |
+
transformers==5.8.1
|
| 102 |
+
typer==0.25.1
|
| 103 |
+
typing-inspection==0.4.2
|
| 104 |
+
typing_extensions==4.15.0
|
| 105 |
+
uritemplate==4.2.0
|
| 106 |
+
urllib3==2.7.0
|
| 107 |
+
uvicorn==0.47.0
|
| 108 |
+
wasabi==1.1.3
|
| 109 |
+
weasel==1.0.0
|
| 110 |
+
wrapt==2.1.2
|
.gitignore
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Byte-compiled / optimized / DLL files
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[codz]
|
| 4 |
+
*$py.class
|
| 5 |
+
|
| 6 |
+
# C extensions
|
| 7 |
+
*.so
|
| 8 |
+
|
| 9 |
+
# Distribution / packaging
|
| 10 |
+
.Python
|
| 11 |
+
build/
|
| 12 |
+
develop-eggs/
|
| 13 |
+
dist/
|
| 14 |
+
downloads/
|
| 15 |
+
eggs/
|
| 16 |
+
.eggs/
|
| 17 |
+
lib/
|
| 18 |
+
lib64/
|
| 19 |
+
parts/
|
| 20 |
+
sdist/
|
| 21 |
+
var/
|
| 22 |
+
wheels/
|
| 23 |
+
share/python-wheels/
|
| 24 |
+
*.egg-info/
|
| 25 |
+
.installed.cfg
|
| 26 |
+
*.egg
|
| 27 |
+
MANIFEST
|
| 28 |
+
|
| 29 |
+
# PyInstaller
|
| 30 |
+
# Usually these files are written by a python script from a template
|
| 31 |
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
| 32 |
+
*.manifest
|
| 33 |
+
*.spec
|
| 34 |
+
|
| 35 |
+
# Installer logs
|
| 36 |
+
pip-log.txt
|
| 37 |
+
pip-delete-this-directory.txt
|
| 38 |
+
|
| 39 |
+
# Unit test / coverage reports
|
| 40 |
+
htmlcov/
|
| 41 |
+
.tox/
|
| 42 |
+
.nox/
|
| 43 |
+
.coverage
|
| 44 |
+
.coverage.*
|
| 45 |
+
.cache
|
| 46 |
+
nosetests.xml
|
| 47 |
+
coverage.xml
|
| 48 |
+
*.cover
|
| 49 |
+
*.py.cover
|
| 50 |
+
*.lcov
|
| 51 |
+
.hypothesis/
|
| 52 |
+
.pytest_cache/
|
| 53 |
+
cover/
|
| 54 |
+
|
| 55 |
+
# Translations
|
| 56 |
+
*.mo
|
| 57 |
+
*.pot
|
| 58 |
+
|
| 59 |
+
# Django stuff:
|
| 60 |
+
*.log
|
| 61 |
+
local_settings.py
|
| 62 |
+
db.sqlite3
|
| 63 |
+
db.sqlite3-journal
|
| 64 |
+
|
| 65 |
+
# Flask stuff:
|
| 66 |
+
instance/
|
| 67 |
+
.webassets-cache
|
| 68 |
+
|
| 69 |
+
# Scrapy stuff:
|
| 70 |
+
.scrapy
|
| 71 |
+
|
| 72 |
+
# Sphinx documentation
|
| 73 |
+
docs/_build/
|
| 74 |
+
|
| 75 |
+
# PyBuilder
|
| 76 |
+
.pybuilder/
|
| 77 |
+
target/
|
| 78 |
+
|
| 79 |
+
# Jupyter Notebook
|
| 80 |
+
.ipynb_checkpoints
|
| 81 |
+
|
| 82 |
+
# IPython
|
| 83 |
+
profile_default/
|
| 84 |
+
ipython_config.py
|
| 85 |
+
|
| 86 |
+
# pyenv
|
| 87 |
+
# For a library or package, you might want to ignore these files since the code is
|
| 88 |
+
# intended to run in multiple environments; otherwise, check them in:
|
| 89 |
+
# .python-version
|
| 90 |
+
|
| 91 |
+
# pipenv
|
| 92 |
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
| 93 |
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
| 94 |
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
| 95 |
+
# install all needed dependencies.
|
| 96 |
+
# Pipfile.lock
|
| 97 |
+
|
| 98 |
+
# UV
|
| 99 |
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
| 100 |
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
| 101 |
+
# commonly ignored for libraries.
|
| 102 |
+
# uv.lock
|
| 103 |
+
|
| 104 |
+
# poetry
|
| 105 |
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
| 106 |
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
| 107 |
+
# commonly ignored for libraries.
|
| 108 |
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
| 109 |
+
# poetry.lock
|
| 110 |
+
# poetry.toml
|
| 111 |
+
|
| 112 |
+
# pdm
|
| 113 |
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
| 114 |
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
| 115 |
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
| 116 |
+
# pdm.lock
|
| 117 |
+
# pdm.toml
|
| 118 |
+
.pdm-python
|
| 119 |
+
.pdm-build/
|
| 120 |
+
|
| 121 |
+
# pixi
|
| 122 |
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
| 123 |
+
# pixi.lock
|
| 124 |
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
| 125 |
+
# in the .venv directory. It is recommended not to include this directory in version control.
|
| 126 |
+
.pixi/*
|
| 127 |
+
!.pixi/config.toml
|
| 128 |
+
|
| 129 |
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
| 130 |
+
__pypackages__/
|
| 131 |
+
|
| 132 |
+
# Celery stuff
|
| 133 |
+
celerybeat-schedule*
|
| 134 |
+
celerybeat.pid
|
| 135 |
+
|
| 136 |
+
# Redis
|
| 137 |
+
*.rdb
|
| 138 |
+
*.aof
|
| 139 |
+
*.pid
|
| 140 |
+
|
| 141 |
+
# RabbitMQ
|
| 142 |
+
mnesia/
|
| 143 |
+
rabbitmq/
|
| 144 |
+
rabbitmq-data/
|
| 145 |
+
|
| 146 |
+
# ActiveMQ
|
| 147 |
+
activemq-data/
|
| 148 |
+
|
| 149 |
+
# SageMath parsed files
|
| 150 |
+
*.sage.py
|
| 151 |
+
|
| 152 |
+
# Environments
|
| 153 |
+
.env
|
| 154 |
+
.envrc
|
| 155 |
+
.venv
|
| 156 |
+
env/
|
| 157 |
+
venv/
|
| 158 |
+
ENV/
|
| 159 |
+
env.bak/
|
| 160 |
+
venv.bak/
|
| 161 |
+
|
| 162 |
+
# Spyder project settings
|
| 163 |
+
.spyderproject
|
| 164 |
+
.spyproject
|
| 165 |
+
|
| 166 |
+
# Rope project settings
|
| 167 |
+
.ropeproject
|
| 168 |
+
|
| 169 |
+
# mkdocs documentation
|
| 170 |
+
/site
|
| 171 |
+
|
| 172 |
+
# mypy
|
| 173 |
+
.mypy_cache/
|
| 174 |
+
.dmypy.json
|
| 175 |
+
dmypy.json
|
| 176 |
+
|
| 177 |
+
# Pyre type checker
|
| 178 |
+
.pyre/
|
| 179 |
+
|
| 180 |
+
# pytype static type analyzer
|
| 181 |
+
.pytype/
|
| 182 |
+
|
| 183 |
+
# Cython debug symbols
|
| 184 |
+
cython_debug/
|
| 185 |
+
|
| 186 |
+
# PyCharm
|
| 187 |
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
| 188 |
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
| 189 |
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
| 190 |
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
| 191 |
+
# .idea/
|
| 192 |
+
|
| 193 |
+
# Abstra
|
| 194 |
+
# Abstra is an AI-powered process automation framework.
|
| 195 |
+
# Ignore directories containing user credentials, local state, and settings.
|
| 196 |
+
# Learn more at https://abstra.io/docs
|
| 197 |
+
.abstra/
|
| 198 |
+
|
| 199 |
+
# Visual Studio Code
|
| 200 |
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
| 201 |
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
| 202 |
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
| 203 |
+
# you could uncomment the following to ignore the entire vscode folder
|
| 204 |
+
# .vscode/
|
| 205 |
+
# Temporary file for partial code execution
|
| 206 |
+
tempCodeRunnerFile.py
|
| 207 |
+
|
| 208 |
+
# Ruff stuff:
|
| 209 |
+
.ruff_cache/
|
| 210 |
+
|
| 211 |
+
# PyPI configuration file
|
| 212 |
+
.pypirc
|
| 213 |
+
|
| 214 |
+
# Marimo
|
| 215 |
+
marimo/_static/
|
| 216 |
+
marimo/_lsp/
|
| 217 |
+
__marimo__/
|
| 218 |
+
|
| 219 |
+
# Streamlit
|
| 220 |
+
.streamlit/secrets.toml
|
| 221 |
+
|
| 222 |
+
.all-installed-packages
|
| 223 |
+
**/.DS_Store
|
| 224 |
+
archive/
|
| 225 |
+
lolaby.gguf
|
| 226 |
+
lolaby-llama-3b/
|
| 227 |
+
.venv-arm64/
|
| 228 |
+
.venv-spike/
|
| 229 |
+
assets/pipeline.png
|
LICENSE
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Apache License
|
| 2 |
+
Version 2.0, January 2004
|
| 3 |
+
http://www.apache.org/licenses/
|
| 4 |
+
|
| 5 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 6 |
+
|
| 7 |
+
1. Definitions.
|
| 8 |
+
|
| 9 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 10 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 11 |
+
|
| 12 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 13 |
+
the copyright owner that is granting the License.
|
| 14 |
+
|
| 15 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 16 |
+
other entities that control, are controlled by, or are under common
|
| 17 |
+
control with that entity. For the purposes of this definition,
|
| 18 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 19 |
+
direction or management of such entity, whether by contract or
|
| 20 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 21 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 22 |
+
|
| 23 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 24 |
+
exercising permissions granted by this License.
|
| 25 |
+
|
| 26 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 27 |
+
including but not limited to software source code, documentation
|
| 28 |
+
source, and configuration files.
|
| 29 |
+
|
| 30 |
+
"Object" form shall mean any form resulting from mechanical
|
| 31 |
+
transformation or translation of a Source form, including but
|
| 32 |
+
not limited to compiled object code, generated documentation,
|
| 33 |
+
and conversions to other media types.
|
| 34 |
+
|
| 35 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 36 |
+
Object form, made available under the License, as indicated by a
|
| 37 |
+
copyright notice that is included in or attached to the work
|
| 38 |
+
(an example is provided in the Appendix below).
|
| 39 |
+
|
| 40 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 41 |
+
form, that is based on (or derived from) the Work and for which the
|
| 42 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 43 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 44 |
+
of this License, Derivative Works shall not include works that remain
|
| 45 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 46 |
+
the Work and Derivative Works thereof.
|
| 47 |
+
|
| 48 |
+
"Contribution" shall mean any work of authorship, including
|
| 49 |
+
the original version of the Work and any modifications or additions
|
| 50 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 51 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 52 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 53 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 54 |
+
means any form of electronic, verbal, or written communication sent
|
| 55 |
+
to the Licensor or its representatives, including but not limited to
|
| 56 |
+
communication on electronic mailing lists, source code control systems,
|
| 57 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 58 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 59 |
+
excluding communication that is conspicuously marked or otherwise
|
| 60 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 61 |
+
|
| 62 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 63 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 64 |
+
subsequently incorporated within the Work.
|
| 65 |
+
|
| 66 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 67 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 68 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 69 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 70 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 71 |
+
Work and such Derivative Works in Source or Object form.
|
| 72 |
+
|
| 73 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 74 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 75 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 76 |
+
(except as stated in this section) patent license to make, have made,
|
| 77 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 78 |
+
where such license applies only to those patent claims licensable
|
| 79 |
+
by such Contributor that are necessarily infringed by their
|
| 80 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 81 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 82 |
+
institute patent litigation against any entity (including a
|
| 83 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 84 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 85 |
+
or contributory patent infringement, then any patent licenses
|
| 86 |
+
granted to You under this License for that Work shall terminate
|
| 87 |
+
as of the date such litigation is filed.
|
| 88 |
+
|
| 89 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 90 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 91 |
+
modifications, and in Source or Object form, provided that You
|
| 92 |
+
meet the following conditions:
|
| 93 |
+
|
| 94 |
+
(a) You must give any other recipients of the Work or
|
| 95 |
+
Derivative Works a copy of this License; and
|
| 96 |
+
|
| 97 |
+
(b) You must cause any modified files to carry prominent notices
|
| 98 |
+
stating that You changed the files; and
|
| 99 |
+
|
| 100 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 101 |
+
that You distribute, all copyright, patent, trademark, and
|
| 102 |
+
attribution notices from the Source form of the Work,
|
| 103 |
+
excluding those notices that do not pertain to any part of
|
| 104 |
+
the Derivative Works; and
|
| 105 |
+
|
| 106 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 107 |
+
distribution, then any Derivative Works that You distribute must
|
| 108 |
+
include a readable copy of the attribution notices contained
|
| 109 |
+
within such NOTICE file, excluding those notices that do not
|
| 110 |
+
pertain to any part of the Derivative Works, in at least one
|
| 111 |
+
of the following places: within a NOTICE text file distributed
|
| 112 |
+
as part of the Derivative Works; within the Source form or
|
| 113 |
+
documentation, if provided along with the Derivative Works; or,
|
| 114 |
+
within a display generated by the Derivative Works, if and
|
| 115 |
+
wherever such third-party notices normally appear. The contents
|
| 116 |
+
of the NOTICE file are for informational purposes only and
|
| 117 |
+
do not modify the License. You may add Your own attribution
|
| 118 |
+
notices within Derivative Works that You distribute, alongside
|
| 119 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 120 |
+
that such additional attribution notices cannot be construed
|
| 121 |
+
as modifying the License.
|
| 122 |
+
|
| 123 |
+
You may add Your own copyright statement to Your modifications and
|
| 124 |
+
may provide additional or different license terms and conditions
|
| 125 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 126 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 127 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 128 |
+
the conditions stated in this License.
|
| 129 |
+
|
| 130 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 131 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 132 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 133 |
+
this License, without any additional terms or conditions.
|
| 134 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 135 |
+
the terms of any separate license agreement you may have executed
|
| 136 |
+
with Licensor regarding such Contributions.
|
| 137 |
+
|
| 138 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 139 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 140 |
+
except as required for describing the origin of the Work and
|
| 141 |
+
reproducing the content of the NOTICE file.
|
| 142 |
+
|
| 143 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 144 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 145 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 146 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 147 |
+
implied, including, without limitation, any warranties or conditions
|
| 148 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 149 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 150 |
+
appropriateness of using or redistributing the Work and assume any
|
| 151 |
+
risks associated with Your exercise of permissions under this License.
|
| 152 |
+
|
| 153 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 154 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 155 |
+
unless required by applicable law (such as deliberate and grossly
|
| 156 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 157 |
+
liable to You for damages, including any direct, indirect, special,
|
| 158 |
+
incidental, or consequential damages of any character arising as a
|
| 159 |
+
result of this License or out of the use or inability to use the
|
| 160 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 161 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 162 |
+
other commercial damages or losses), even if such Contributor
|
| 163 |
+
has been advised of the possibility of such damages.
|
| 164 |
+
|
| 165 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 166 |
+
the Work or Derivative Works thereof, You may accept and charge a
|
| 167 |
+
fee for, or offer to, accept support, warranty, indemnity, or
|
| 168 |
+
other liability obligations and/or rights consistent with this
|
| 169 |
+
License. However, in accepting such obligations, You may act only
|
| 170 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 171 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 172 |
+
defend, and hold each Contributor harmless for any liability
|
| 173 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 174 |
+
of your accepting any such warranty or additional liability.
|
| 175 |
+
|
| 176 |
+
END OF TERMS AND CONDITIONS
|
| 177 |
+
|
| 178 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 179 |
+
|
| 180 |
+
To apply the Apache License to your work, attach the following
|
| 181 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 182 |
+
replaced with your own identifying information. (Don't include
|
| 183 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 184 |
+
comment syntax for the file format. We also recommend that a
|
| 185 |
+
file or class name and description of purpose be included on the
|
| 186 |
+
same "printed page" as the copyright notice for easier
|
| 187 |
+
identification within third-party archives.
|
| 188 |
+
|
| 189 |
+
Copyright 2026 AndrΓ© Oliveira.
|
| 190 |
+
|
| 191 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 192 |
+
you may not use this file except in compliance with the License.
|
| 193 |
+
You may obtain a copy of the License at
|
| 194 |
+
|
| 195 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 196 |
+
|
| 197 |
+
Unless required by applicable law or agreed to in writing, software
|
| 198 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 199 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 200 |
+
See the License for the specific language governing permissions and
|
| 201 |
+
limitations under the License.
|
| 202 |
+
|
| 203 |
+
----------------------------------------------------------------------
|
| 204 |
+
Note: This LICENSE covers the Lolaby APPLICATION CODE in this repository
|
| 205 |
+
only (Gradio app, audio synths, vision wrapper, safety filter, etc.).
|
| 206 |
+
|
| 207 |
+
The fine-tuned lyric MODEL referenced by this app lives in a separate
|
| 208 |
+
Hugging Face repository and is distributed under the Llama 3.2 Community
|
| 209 |
+
License. See that repository's LICENSE file for those terms.
|
README.md
CHANGED
|
@@ -1,17 +1,158 @@
|
|
| 1 |
---
|
| 2 |
title: Lolaby
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version:
|
| 8 |
-
python_version: '3.13'
|
| 9 |
app_file: app.py
|
| 10 |
-
pinned:
|
| 11 |
-
license:
|
| 12 |
-
short_description: AI-powered lullabies
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
---
|
| 14 |
|
| 15 |
-
|
| 16 |
|
| 17 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
title: Lolaby
|
| 3 |
+
emoji: π
|
| 4 |
+
colorFrom: yellow
|
| 5 |
+
colorTo: blue
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: "5.0.0"
|
|
|
|
| 8 |
app_file: app.py
|
| 9 |
+
pinned: true
|
| 10 |
+
license: llama3.2
|
| 11 |
+
short_description: AI-powered lullabies.
|
| 12 |
+
tags:
|
| 13 |
+
- lullaby
|
| 14 |
+
- children
|
| 15 |
+
- small-model
|
| 16 |
+
- llama-cpp
|
| 17 |
+
- fine-tuned
|
| 18 |
+
- on-device
|
| 19 |
+
- hackathon
|
| 20 |
+
- build-small-2026
|
| 21 |
+
- backyard-ai
|
| 22 |
---
|
| 23 |
|
| 24 |
+
# Lolaby β AI-powered lullabies
|
| 25 |
|
| 26 |
+
> *A tiny AI that watches your child's drawings and sings them a personalised lullaby.*
|
| 27 |
+
|
| 28 |
+
Built for the [Hugging Face **Build Small Hackathon 2026**](https://huggingface.co/build-small-hackathon) Β· **Backyard AI** track π‘
|
| 29 |
+
|
| 30 |
+
**Try it:** [the live Space](https://huggingface.co/spaces/build-small-hackathon/lolaby) Β· **Demo video:** [60-second walkthrough](placeholder_demo_video_url) Β· **Model:** [`lolaby-llama-3b` on the Hub](https://huggingface.co/build-small-hackathon/lolaby-llama-3b) Β· **Field notes:** [the build journal](placeholder_blog_url)
|
| 31 |
+
|
| 32 |
+
---
|
| 33 |
+
|
| 34 |
+
## The Problem
|
| 35 |
+
|
| 36 |
+
My partner's sister is a kindergarten teacher, who runs nap time for fifteen 4-year-olds on a daily basis. Every child wants the same thing at the start of nap: *"sing me a song with my name in it."* She'd love to give each child their own song, drawn from what they actually love that week (a stuffed fox, a new puppy, the rainbow). She doesn't have time, musical expertise nor a tool.
|
| 37 |
+
|
| 38 |
+
**Lolaby is the tool.** The child shows Lola what they love β either by doodling on screen, or by handing the teacher a paper drawing for her to snap a photo of. The teacher types their name. A small, on-device AI watches the drawing, writes them a lullaby about it, and sings it back β in about a minute.
|
| 39 |
+
|
| 40 |
+
Everything runs locally. No cloud LLM, no per-song API cost, no children's drawings or names ever leaving the device.
|
| 41 |
+
|
| 42 |
+
## How it works
|
| 43 |
+
|
| 44 |
+

|
| 45 |
+
|
| 46 |
+
**The drawing is optional.**
|
| 47 |
+
No canvas drawing AND no upload? β’ The form's *What do they love?* field is used instead.
|
| 48 |
+
Drawing AND typed loves? β’ Both inform the song.
|
| 49 |
+
|
| 50 |
+
*Lola* (Lolaby's AI character) tells you what she saw between the audio and the lyrics, so you can see how the drawing turned into the song.
|
| 51 |
+
|
| 52 |
+
## What's inside
|
| 53 |
+
|
| 54 |
+
| Component | Model / Library | Where it runs |
|
| 55 |
+
| ---------------- | -------------------------------------------------- | ------------- |
|
| 56 |
+
| Lyric generation | **Llama 3.2 3B**, fine-tuned, via `llama.cpp` | CPU, locally |
|
| 57 |
+
| Drawing β words | **MiniCPM-V 4.6** (1.3B) via `transformers` | GPU if available, and CPU otherwise (locally) |
|
| 58 |
+
| Stroke fallback | Pure NumPy color/density analysis | CPU, locally |
|
| 59 |
+
| Singing voice | **Kokoro 82M** | CPU, locally |
|
| 60 |
+
| Instruments | Custom DSP synths, built from spectral analysis | CPU, locally |
|
| 61 |
+
| Content safety | Local keyword + intent filter | CPU, locally |
|
| 62 |
+
|
| 63 |
+
**Total parameters across the whole pipeline: well under 32B.** The lyric model is 3B; the vision model is 1.3B; Kokoro is 82M. The synths have zero parameters β they're DSP (Digital Signal Processors).
|
| 64 |
+
|
| 65 |
+
## Badges
|
| 66 |
+
|
| 67 |
+
This submission is satisties all six hackathon bonus quests:
|
| 68 |
+
|
| 69 |
+
- π **Off the Grid** β **No cloud APIs at runtime.** Every model in the deployed app runs locally. The Anthropic API was used *only at build time* to distill the lyric training dataset (see "Dataset and provenance" below); it is not called when a song is generated.
|
| 70 |
+
- π― **Well-Tuned** β The lyric model is a custom fine-tune of Llama 3.2 3B, [published on the Hub](https://huggingface.co/build-small-hackathon/lolaby-llama-3b). Trained on a 1,500-example dataset built from scratch with mechanical anti-boilerplate gates; 99.4% line uniqueness across the lyric corpus.
|
| 71 |
+
- π¨ **Off-Brand** β Custom Gradio UI: hand-drawn aesthetic with crayon palette, Fredoka One + Patrick Hand typography, paper-textured cards, character voice via the "Lola" persona that tells you what she saw.
|
| 72 |
+
- π¦ **Llama Champion** β Lyric inference runs through `llama-cpp-python` (the `llama.cpp` Python binding) on CPU at Q4_K_M quantisation.
|
| 73 |
+
- π‘ **Sharing is Caring** β A full generation trace (drawing β vision β lyric prompt β raw model output β audio render parameters) is [published on the Hub](placeholder_trace_url) so anyone can study how the pipeline composes.
|
| 74 |
+
- π **Field Notes** β [Build journal here](placeholder_blog_url): the dataset rebuild, the DSP-from-spectral-analysis decision, the vision-vs-strokes architecture, and what didn't work.
|
| 75 |
+
|
| 76 |
+
## Demo
|
| 77 |
+
|
| 78 |
+
β **[60-second walkthrough](placeholder_demo_video_url)** β a child draws, a teacher types, Lola sings.
|
| 79 |
+
|
| 80 |
+
β **[Social media post](placeholder_social_post_url)** β for sharing.
|
| 81 |
+
|
| 82 |
+
## Try it yourself
|
| 83 |
+
|
| 84 |
+
Web version: **[the live Space](https://huggingface.co/spaces/build-small-hackathon/lolaby)**
|
| 85 |
+
|
| 86 |
+
Or to run it locally:
|
| 87 |
+
|
| 88 |
+
```bash
|
| 89 |
+
git clone https://huggingface.co/spaces/build-small-hackathon/lolaby
|
| 90 |
+
cd lolaby
|
| 91 |
+
pip install -r requirements.txt
|
| 92 |
+
python app.py
|
| 93 |
+
```
|
| 94 |
+
|
| 95 |
+
The lyric model is fetched from the Hub on first run (~2 GB GGUF, cached after).
|
| 96 |
+
|
| 97 |
+
## Dataset
|
| 98 |
+
|
| 99 |
+
The lyric model was fine-tuned on **1,500 lullabies distilled from Claude Haiku 4.5** with strict anti-boilerplate gates: per-line n-gram dedup, opener dedup, theme caps, format gates, and per-example safety screening. The dataset itself, the generator (`generate_dataset.py`), and the training notebook (`train_lullaby.ipynb`) are in the [`train/`](./train/) folder. Regenerating the dataset requires an Anthropic API key; running the deployed app does not.
|
| 100 |
+
|
| 101 |
+
Full details in the [model card](https://huggingface.co/build-small-hackathon/lolaby-llama-3b) and the [Field Notes blog post](placeholder_blog_url).
|
| 102 |
+
|
| 103 |
+
## Repository
|
| 104 |
+
|
| 105 |
+
```
|
| 106 |
+
.
|
| 107 |
+
βββ app.py # Gradio entrypoint
|
| 108 |
+
βββ requirements.txt
|
| 109 |
+
βββ utils/
|
| 110 |
+
β βββ safety.py # Content-safety filter
|
| 111 |
+
βββ draw/ # Drawing-related helpers
|
| 112 |
+
β βββ vision.py # MiniCPM-V 4.6 wrapper
|
| 113 |
+
β βββ strokes.py # NumPy fallback if vision is unavailable
|
| 114 |
+
βββ synths/ # Custom DSP instruments + Kokoro voice
|
| 115 |
+
β βββ guitar.py
|
| 116 |
+
β βββ piano.py
|
| 117 |
+
β βββ musicbox.py
|
| 118 |
+
β βββ harp.py
|
| 119 |
+
β βββ xylophone.py
|
| 120 |
+
β βββ ocarina.py
|
| 121 |
+
β βββ voice.py
|
| 122 |
+
βββ train/ # Dataset + training (build-time only; not used at runtime)
|
| 123 |
+
β βββ generate_dataset.py
|
| 124 |
+
β βββ train_lullaby.ipynb
|
| 125 |
+
β βββ lullaby_dataset.jsonl
|
| 126 |
+
βββ tests/ # Sanity checks for audio + LLM + voice
|
| 127 |
+
```
|
| 128 |
+
|
| 129 |
+
## Safety
|
| 130 |
+
|
| 131 |
+
Lolaby is built for small children, so safety isn't an afterthought β it's wired through the pipeline at three points:
|
| 132 |
+
|
| 133 |
+
- **At training time** β every example in the lyric model's training set was screened during dataset distillation. The model learned from already-wholesome material, not from the open web.
|
| 134 |
+
- **At input time** β when a user types a *love* or *fear*, the text is screened against a curated list of terms inappropriate for a child's lullaby (death, weapons, horror, substances, self-harm). Anything matching gets a gentle redirect ("Let's keep the lullaby to gentle, cosy thingsβ¦") instead of a generation.
|
| 135 |
+
- **At generation time** β the same filter is shared between the runtime app and the dataset generator, so the training data and the live app can never enforce different rules.
|
| 136 |
+
|
| 137 |
+
## Limitations
|
| 138 |
+
|
| 139 |
+
- **English only.** The lyric model was trained on English data.
|
| 140 |
+
- **First-time Space cold-start** might take some time while all models pre-warm at boot.
|
| 141 |
+
- **Strange or unusual loves** may be gently generalised by the lyric model into a nearby comforting concept β that's a deliberate behaviour for a bedtime song (soft landing > literal lookup) and described in the model card.
|
| 142 |
+
- **Children's drawings are interpreted by an AI** that maps rough shapes and colours onto familiar concepts β much like an adult parent guessing what a kid drew. It can miss subtle details: a stick figure becomes "a little person", a wobbly square with a triangle becomes "a house". The "Lola sawβ¦" hint shows exactly what she understood, so you can redraw or use the What do they love? field instead.
|
| 143 |
+
|
| 144 |
+
## Credits
|
| 145 |
+
|
| 146 |
+
- **[Meta](https://ai.meta.com/llama/)** β Llama 3.2 3B Instruct (base model, used under the Llama 3.2 Community License).
|
| 147 |
+
- **[OpenBMB](https://huggingface.co/openbmb)** β MiniCPM-V 4.6 (vision).
|
| 148 |
+
- **[hexgrad/Kokoro-82M](https://huggingface.co/hexgrad/Kokoro-82M)** β TTS voice.
|
| 149 |
+
- **[Unsloth](https://github.com/unslothai/unsloth)** β 4-bit + LoRA training stack.
|
| 150 |
+
- **[Hugging Face & Gradio](https://huggingface.co/build-small-hackathon)** β for hosting the Build Small Hackathon and creating space for small-AI projects.
|
| 151 |
+
|
| 152 |
+
## License
|
| 153 |
+
|
| 154 |
+
Apache 2.0 for the app code in this repo. The lyric model weights inherit the **Llama 3.2 Community License** from their base β see the [model card](https://huggingface.co/build-small-hackathon/lolaby-llama-3b) for full terms.
|
| 155 |
+
|
| 156 |
+
---
|
| 157 |
+
|
| 158 |
+
*Lolaby β AI-powered lullabies*
|
app.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
assets/guitar.png
ADDED
|
assets/harp.png
ADDED
|
assets/music-box.png
ADDED
|
assets/ocarina.png
ADDED
|
assets/piano.png
ADDED
|
assets/xylophone.png
ADDED
|
draw/__init__.py
ADDED
|
File without changes
|
draw/strokes.py
ADDED
|
@@ -0,0 +1,278 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
strokes.py β Drawing β "loves" phrase, using stroke/color analysis only.
|
| 3 |
+
|
| 4 |
+
WHY THIS EXISTS
|
| 5 |
+
---------------
|
| 6 |
+
The drawing-input flow wants to feed something into the lullaby pipeline's
|
| 7 |
+
"loves" field. A real vision-language model (MiniCPM-V) can do this richly,
|
| 8 |
+
but it's slow on CPU (~4 minutes per inference on M1) and prohibitive on a
|
| 9 |
+
free CPU Space. This module is the fast, no-model fallback: it reads the
|
| 10 |
+
DRAWING ITSELF as structured data β colors, density, composition, energy β
|
| 11 |
+
and turns that into evocative themes that feel like things the child might
|
| 12 |
+
love.
|
| 13 |
+
|
| 14 |
+
DESIGN PRINCIPLES
|
| 15 |
+
-----------------
|
| 16 |
+
- Honest about what it sees: doesn't claim to recognize objects. It reads
|
| 17 |
+
MOOD (warm/cool/bright/quiet/energetic/calm) and turns that into
|
| 18 |
+
thematic words.
|
| 19 |
+
- Sub-second on CPU. Pure numpy/PIL math, zero models.
|
| 20 |
+
- Varied output: same drawing twice should give slightly different phrases.
|
| 21 |
+
Built-in randomness keeps it from feeling deterministic/canned.
|
| 22 |
+
- Always safe: outputs are drawn from a hand-curated wholesome vocabulary,
|
| 23 |
+
so no chance of producing unsafe themes from raw pixels.
|
| 24 |
+
- Graceful with empty/sparse drawings: returns a gentle default rather than
|
| 25 |
+
crashing or producing weird output.
|
| 26 |
+
|
| 27 |
+
PUBLIC INTERFACE
|
| 28 |
+
----------------
|
| 29 |
+
analyze_drawing(image_pil_or_array) -> dict:
|
| 30 |
+
{
|
| 31 |
+
"loves": str, # phrase to feed the lullaby pipeline
|
| 32 |
+
"mood": str, # human-readable mood label (for UI hint)
|
| 33 |
+
"summary": str, # one-line explanation of what was "seen"
|
| 34 |
+
}
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
import random
|
| 38 |
+
|
| 39 |
+
import numpy as np
|
| 40 |
+
from PIL import Image
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
# Lightweight color buckets. Each bucket carries a vocabulary of WHOLESOME
|
| 44 |
+
# themes a lullaby could weave in. The exact word picked per generation is
|
| 45 |
+
# randomized so the same drawing produces varied (but on-mood) lyrics.
|
| 46 |
+
COLOR_THEMES = {
|
| 47 |
+
"yellow": ["the warm sun", "morning light", "honey and bees", "a glowing lantern",
|
| 48 |
+
"buttercups in the field", "a candle on the windowsill"],
|
| 49 |
+
"orange": ["a cosy fire", "autumn leaves", "a pumpkin lantern", "the sunset",
|
| 50 |
+
"ripe apricots", "a soft glowing lamp"],
|
| 51 |
+
"red": ["red apples", "a robin in the snow", "a little red boat",
|
| 52 |
+
"a tomato in the garden", "a brave red balloon"],
|
| 53 |
+
"pink": ["soft pink clouds", "cherry blossom", "a baby rabbit",
|
| 54 |
+
"the first roses", "watermelon slices", "a gentle flamingo"],
|
| 55 |
+
"purple": ["lavender in the breeze", "the twilight sky", "soft plums",
|
| 56 |
+
"violets in the grass", "an amethyst pebble"],
|
| 57 |
+
"blue": ["the deep ocean", "the night sky", "raindrops on the window",
|
| 58 |
+
"a sleepy whale", "a still pond", "bluebells"],
|
| 59 |
+
"cyan": ["the morning sky", "a gentle stream", "a glass of cool water",
|
| 60 |
+
"the misty lake", "robin-egg blue"],
|
| 61 |
+
"green": ["the meadow grass", "tall pine trees", "a quiet forest",
|
| 62 |
+
"a curled-up turtle", "moss on stones", "lily pads"],
|
| 63 |
+
"brown": ["a sleepy bear", "warm bread", "the trunk of the old oak",
|
| 64 |
+
"a wooden rocking horse", "fresh-turned earth", "a hedgehog"],
|
| 65 |
+
"black": ["the gentle night", "a sleeping cat", "the soft dark",
|
| 66 |
+
"shadows that protect", "a peaceful crow"],
|
| 67 |
+
"white": ["fresh snow", "soft clouds", "a white rabbit", "a paper crane",
|
| 68 |
+
"milk in a small cup", "a swan on the pond"],
|
| 69 |
+
"grey": ["a soft grey rabbit", "the misty morning", "a sleeping mouse",
|
| 70 |
+
"smooth river stones", "a quiet pigeon"],
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
# Density/coverage describes how much of the canvas the child filled.
|
| 74 |
+
DENSITY_WORDS = {
|
| 75 |
+
"sparse": ["the open sky", "a wide quiet field", "a tiny world to discover"],
|
| 76 |
+
"medium": ["a friendly little scene", "a small backyard adventure"],
|
| 77 |
+
"dense": ["a busy garden", "a whole world packed with friends"],
|
| 78 |
+
}
|
| 79 |
+
|
| 80 |
+
# Stroke energy: smooth long strokes vs. many short scribbles β calm vs lively
|
| 81 |
+
ENERGY_WORDS = {
|
| 82 |
+
"calm": ["a peaceful afternoon", "everything moving slowly",
|
| 83 |
+
"a gentle, drowsy day"],
|
| 84 |
+
"lively": ["a joyful, busy day", "everything dancing and playing",
|
| 85 |
+
"a wide-awake adventure that's winding down now"],
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _empty_result():
|
| 90 |
+
"""Sentinel for an empty/near-blank canvas. We return None so callers
|
| 91 |
+
can distinguish 'nothing was drawn' from 'something was drawn that I
|
| 92 |
+
interpreted'. The wrapper in app.py uses this to fall back to whatever
|
| 93 |
+
the user typed in the loves field."""
|
| 94 |
+
return None
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def _to_rgba_array(img):
|
| 98 |
+
"""Accept PIL Image OR numpy array (incl. Gradio Sketchpad's dict-of-layers),
|
| 99 |
+
return an (H, W, 4) uint8 numpy array."""
|
| 100 |
+
# Gradio's ImageEditor/Sketchpad returns a dict {'background': ..., 'layers':
|
| 101 |
+
# [...], 'composite': ...}. We can't use "a or b or c" to pick the first
|
| 102 |
+
# non-None β for numpy arrays that evaluates `bool(array)` and raises
|
| 103 |
+
# "truth value is ambiguous". Test each key for `None` explicitly.
|
| 104 |
+
if isinstance(img, dict):
|
| 105 |
+
picked = None
|
| 106 |
+
for key in ("composite", "image", "background"):
|
| 107 |
+
v = img.get(key)
|
| 108 |
+
if v is not None:
|
| 109 |
+
picked = v
|
| 110 |
+
break
|
| 111 |
+
if picked is None and img.get("layers"):
|
| 112 |
+
picked = img["layers"][0]
|
| 113 |
+
img = picked
|
| 114 |
+
if img is None:
|
| 115 |
+
return None
|
| 116 |
+
if isinstance(img, np.ndarray):
|
| 117 |
+
arr = img
|
| 118 |
+
else:
|
| 119 |
+
arr = np.asarray(img.convert("RGBA"))
|
| 120 |
+
# Normalize to RGBA
|
| 121 |
+
if arr.ndim == 2: # grayscale
|
| 122 |
+
arr = np.stack([arr] * 3 + [np.full_like(arr, 255)], axis=-1)
|
| 123 |
+
elif arr.shape[-1] == 3: # RGB β add opaque alpha
|
| 124 |
+
a = np.full(arr.shape[:2] + (1,), 255, dtype=arr.dtype)
|
| 125 |
+
arr = np.concatenate([arr, a], axis=-1)
|
| 126 |
+
return arr.astype(np.uint8)
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def _classify_pixel_color(r, g, b):
|
| 130 |
+
"""Map an RGB triple to a coarse color name. Handles white/grey/black
|
| 131 |
+
cleanly so background pixels don't get miscategorized as 'yellow' etc."""
|
| 132 |
+
# Brightness + saturation reasoning (cheap, no HSV conversion)
|
| 133 |
+
mx, mn = max(r, g, b), min(r, g, b)
|
| 134 |
+
brightness = mx
|
| 135 |
+
sat = mx - mn
|
| 136 |
+
if brightness > 240 and sat < 25:
|
| 137 |
+
return "white"
|
| 138 |
+
if brightness < 35:
|
| 139 |
+
return "black"
|
| 140 |
+
if sat < 25:
|
| 141 |
+
return "grey"
|
| 142 |
+
# Now we have a "real" color β bucket by which channel dominates
|
| 143 |
+
if r > 200 and g > 150 and b < 120:
|
| 144 |
+
return "yellow"
|
| 145 |
+
if r > 200 and 80 < g < 180 and b < 120:
|
| 146 |
+
return "orange"
|
| 147 |
+
if r > 180 and g < 120 and b < 120:
|
| 148 |
+
return "red"
|
| 149 |
+
if r > 200 and g < 200 and b > 180:
|
| 150 |
+
return "pink"
|
| 151 |
+
if r > 100 and b > 150 and g < 150:
|
| 152 |
+
return "purple"
|
| 153 |
+
if b > 180 and r < 150:
|
| 154 |
+
return "blue"
|
| 155 |
+
if b > 150 and g > 150 and r < 180:
|
| 156 |
+
return "cyan"
|
| 157 |
+
if g > 150 and r < 180 and b < 180:
|
| 158 |
+
return "green"
|
| 159 |
+
if 100 < r < 200 and 60 < g < 140 and b < 100:
|
| 160 |
+
return "brown"
|
| 161 |
+
# default: pick closest of the saturated buckets by dominant channel
|
| 162 |
+
if r >= g and r >= b:
|
| 163 |
+
return "red"
|
| 164 |
+
if g >= b:
|
| 165 |
+
return "green"
|
| 166 |
+
return "blue"
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def analyze_drawing(image, seed=None):
|
| 170 |
+
"""Read a drawing into a 'loves'-style phrase + mood description.
|
| 171 |
+
|
| 172 |
+
image: PIL.Image, numpy array, or Gradio ImageEditor dict.
|
| 173 |
+
seed: optional RNG seed for reproducible-ish output (mostly for tests).
|
| 174 |
+
"""
|
| 175 |
+
rng = random.Random(seed)
|
| 176 |
+
|
| 177 |
+
arr = _to_rgba_array(image)
|
| 178 |
+
if arr is None or arr.size == 0:
|
| 179 |
+
return _empty_result()
|
| 180 |
+
|
| 181 |
+
h, w = arr.shape[:2]
|
| 182 |
+
rgb = arr[..., :3]
|
| 183 |
+
alpha = arr[..., 3]
|
| 184 |
+
|
| 185 |
+
# Background detection: a pixel counts as "drawn" if it's NOT near-white-opaque
|
| 186 |
+
# and (if transparent canvas) has non-zero alpha. This is intentionally
|
| 187 |
+
# generous β a child's lighter strokes still count.
|
| 188 |
+
drawn_mask = (alpha > 30) & ~(
|
| 189 |
+
(rgb[..., 0] > 240) & (rgb[..., 1] > 240) & (rgb[..., 2] > 240)
|
| 190 |
+
)
|
| 191 |
+
total_pixels = h * w
|
| 192 |
+
drawn_pixels = int(drawn_mask.sum())
|
| 193 |
+
coverage = drawn_pixels / max(1, total_pixels)
|
| 194 |
+
|
| 195 |
+
# Truly empty β gentle default
|
| 196 |
+
if coverage < 0.005: # less than 0.5% drawn
|
| 197 |
+
return _empty_result()
|
| 198 |
+
|
| 199 |
+
# --- Color analysis (sample for speed on large canvases) ---
|
| 200 |
+
drawn_rgb = rgb[drawn_mask]
|
| 201 |
+
if len(drawn_rgb) > 20000:
|
| 202 |
+
idx = np.linspace(0, len(drawn_rgb) - 1, 20000, dtype=int)
|
| 203 |
+
drawn_rgb = drawn_rgb[idx]
|
| 204 |
+
|
| 205 |
+
color_counts = {}
|
| 206 |
+
for px in drawn_rgb:
|
| 207 |
+
c = _classify_pixel_color(int(px[0]), int(px[1]), int(px[2]))
|
| 208 |
+
color_counts[c] = color_counts.get(c, 0) + 1
|
| 209 |
+
|
| 210 |
+
# Drop white/grey/black from the "main color" picks unless they dominate β
|
| 211 |
+
# they're often background-ish even when classified as drawn. But preserve
|
| 212 |
+
# them if they're genuinely the most-used (a deliberate black-night drawing).
|
| 213 |
+
total = sum(color_counts.values())
|
| 214 |
+
weighted = {
|
| 215 |
+
c: (n / total) for c, n in color_counts.items()
|
| 216 |
+
}
|
| 217 |
+
# Pick top 2 colors by share, but boost saturated colors slightly
|
| 218 |
+
BOOST = {"yellow": 1.2, "orange": 1.2, "red": 1.2, "pink": 1.2,
|
| 219 |
+
"purple": 1.2, "blue": 1.1, "cyan": 1.2, "green": 1.1, "brown": 1.1}
|
| 220 |
+
ranked = sorted(weighted.items(),
|
| 221 |
+
key=lambda kv: kv[1] * BOOST.get(kv[0], 1.0),
|
| 222 |
+
reverse=True)
|
| 223 |
+
top_colors = [c for c, _ in ranked if weighted[c] > 0.08][:2]
|
| 224 |
+
if not top_colors:
|
| 225 |
+
top_colors = [ranked[0][0]]
|
| 226 |
+
|
| 227 |
+
# --- Density bucket ---
|
| 228 |
+
if coverage < 0.05:
|
| 229 |
+
density = "sparse"
|
| 230 |
+
elif coverage < 0.25:
|
| 231 |
+
density = "medium"
|
| 232 |
+
else:
|
| 233 |
+
density = "dense"
|
| 234 |
+
|
| 235 |
+
# --- Energy: a cheap proxy is how MANY transitions between drawn and not
|
| 236 |
+
# along scan lines. Lots of transitions = many short scribbles = lively.
|
| 237 |
+
# Few transitions = long smooth strokes = calm. Subsample for speed.
|
| 238 |
+
step = max(1, h // 40)
|
| 239 |
+
transitions = 0
|
| 240 |
+
rows_sampled = 0
|
| 241 |
+
for y in range(0, h, step):
|
| 242 |
+
row = drawn_mask[y]
|
| 243 |
+
transitions += int(np.sum(row[1:] != row[:-1]))
|
| 244 |
+
rows_sampled += 1
|
| 245 |
+
avg_transitions = transitions / max(1, rows_sampled)
|
| 246 |
+
energy = "lively" if avg_transitions > 6 else "calm"
|
| 247 |
+
|
| 248 |
+
# --- Pick the loves phrase ---
|
| 249 |
+
# 1-2 color themes + maybe a density flavor + maybe an energy flavor.
|
| 250 |
+
# Randomize WHICH specific phrase per color so the same drawing varies.
|
| 251 |
+
parts = []
|
| 252 |
+
for c in top_colors:
|
| 253 |
+
if c in COLOR_THEMES:
|
| 254 |
+
parts.append(rng.choice(COLOR_THEMES[c]))
|
| 255 |
+
# 50% chance to add a density or energy color phrase
|
| 256 |
+
if rng.random() < 0.5:
|
| 257 |
+
parts.append(rng.choice(ENERGY_WORDS[energy]))
|
| 258 |
+
elif rng.random() < 0.5:
|
| 259 |
+
parts.append(rng.choice(DENSITY_WORDS[density]))
|
| 260 |
+
|
| 261 |
+
loves = ", ".join(parts) if parts else rng.choice(DENSITY_WORDS["medium"])
|
| 262 |
+
|
| 263 |
+
# --- Human-readable mood label & summary (for the UI) ---
|
| 264 |
+
mood_label_map = {
|
| 265 |
+
("calm", "sparse"): "quiet and dreamy",
|
| 266 |
+
("calm", "medium"): "calm and cosy",
|
| 267 |
+
("calm", "dense"): "rich and peaceful",
|
| 268 |
+
("lively", "sparse"): "playful and light",
|
| 269 |
+
("lively", "medium"): "happy and bright",
|
| 270 |
+
("lively", "dense"): "joyful and busy",
|
| 271 |
+
}
|
| 272 |
+
mood = mood_label_map.get((energy, density), "warm and gentle")
|
| 273 |
+
|
| 274 |
+
color_word = top_colors[0] if top_colors else "soft"
|
| 275 |
+
summary = (f"A {mood} drawing β lots of {color_word}, "
|
| 276 |
+
f"{'lots of small strokes' if energy == 'lively' else 'smooth gentle strokes'}.")
|
| 277 |
+
|
| 278 |
+
return {"loves": loves, "mood": mood, "summary": summary}
|
draw/vision.py
ADDED
|
@@ -0,0 +1,351 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
vision.py β Drawing/photo β "loves" phrase via MiniCPM-V 4.6, with
|
| 3 |
+
stroke-analysis fallback.
|
| 4 |
+
|
| 5 |
+
DEPLOYMENT MODEL (this is the design decision that shapes everything):
|
| 6 |
+
|
| 7 |
+
- On HuggingFace ZeroGPU: the Space runs on CPU. A GPU is attached ONLY
|
| 8 |
+
inside @spaces.GPU-decorated functions, then released. This module's
|
| 9 |
+
`describe_with_vision()` is decorated, so the H200 GPU attaches just for
|
| 10 |
+
the ~few seconds of vision inference, then detaches. The lyric model
|
| 11 |
+
(llama.cpp) stays on CPU the whole time and never sees the GPU.
|
| 12 |
+
|
| 13 |
+
- On local M1 (no GPU): @spaces.GPU is a no-op (documented behavior). The
|
| 14 |
+
same function runs on CPU, slowly (~30-60s for 1.3B at 16x downsample).
|
| 15 |
+
Acceptable for local testing the pipeline end-to-end.
|
| 16 |
+
|
| 17 |
+
- If MiniCPM-V can't load at all (missing deps, oom, etc.), we fall back
|
| 18 |
+
to stroke analysis. The app never breaks β the experience just gets
|
| 19 |
+
cruder.
|
| 20 |
+
|
| 21 |
+
WHY MiniCPM-V 4.6 (1.3B) AND NOT 2.6 (8B):
|
| 22 |
+
- 6x smaller β faster vision step + smaller ZeroGPU footprint per call.
|
| 23 |
+
- Quality is fine for "describe a child's drawing in 8 words" β we are
|
| 24 |
+
not doing OCR on documents.
|
| 25 |
+
- Uses standard transformers AutoModelForImageTextToText API.
|
| 26 |
+
|
| 27 |
+
WHY downsample_mode="16x":
|
| 28 |
+
- 4x divides image into many tokens (finer detail, slower). 16x is faster
|
| 29 |
+
and the right precision for our task: we want themes, not pixels.
|
| 30 |
+
|
| 31 |
+
PUBLIC INTERFACE:
|
| 32 |
+
vision_available() -> bool
|
| 33 |
+
describe_with_vision(image, seed=None) -> dict # uses GPU on ZeroGPU
|
| 34 |
+
describe(image, seed=None) -> dict # auto-picks vision or strokes
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
import os
|
| 38 |
+
import traceback
|
| 39 |
+
|
| 40 |
+
# `spaces` is the HuggingFace ZeroGPU helper. It's a no-op outside HF Spaces,
|
| 41 |
+
# so the decorator is safe to apply everywhere. We import it gracefully so
|
| 42 |
+
# the app still runs on a fresh machine without the package installed.
|
| 43 |
+
try:
|
| 44 |
+
import spaces # noqa: F401
|
| 45 |
+
_HAVE_SPACES = True
|
| 46 |
+
except ImportError:
|
| 47 |
+
# Provide a no-op decorator so @spaces.GPU(...) works locally without
|
| 48 |
+
# the package β mirrors what HF's `spaces` does in non-ZeroGPU envs.
|
| 49 |
+
class _SpacesShim:
|
| 50 |
+
def GPU(self, *args, **kwargs):
|
| 51 |
+
# Support both @spaces.GPU and @spaces.GPU(duration=...)
|
| 52 |
+
if len(args) == 1 and callable(args[0]) and not kwargs:
|
| 53 |
+
return args[0]
|
| 54 |
+
def deco(fn):
|
| 55 |
+
return fn
|
| 56 |
+
return deco
|
| 57 |
+
spaces = _SpacesShim()
|
| 58 |
+
_HAVE_SPACES = False
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
VISION_MODEL_ID = "openbmb/MiniCPM-V-4.6"
|
| 62 |
+
|
| 63 |
+
# Tuned for "child's drawing β short loves phrase". We want concrete nouns, no
|
| 64 |
+
# stylistic description ("crayon", "stick figure"), and safety baked in so the
|
| 65 |
+
# vision model can't surface something dark we'd then have to filter out.
|
| 66 |
+
DESCRIBE_PROMPT = (
|
| 67 |
+
"This is a young child's drawing. In 8 words or fewer, list the main "
|
| 68 |
+
"things you see as simple nouns a parent would recognise (e.g. "
|
| 69 |
+
"'a cat, the sun, a little house'). Only list wholesome, friendly things "
|
| 70 |
+
"suitable for a bedtime lullaby. Do not describe the drawing style β "
|
| 71 |
+
"just the things in it. Do not add commentary."
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
# Lazy globals β load the model ONCE on first call, keep it loaded. On
|
| 76 |
+
# ZeroGPU the model lives in CPU memory between calls and is moved/used on
|
| 77 |
+
# the GPU only inside the decorated function.
|
| 78 |
+
_model = None
|
| 79 |
+
_processor = None
|
| 80 |
+
_load_error = None
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def _try_load():
|
| 84 |
+
"""Best-effort lazy load. Returns True on success, False otherwise.
|
| 85 |
+
On failure, the reason is stashed in _load_error for diagnostics."""
|
| 86 |
+
global _model, _processor, _load_error
|
| 87 |
+
if _model is not None:
|
| 88 |
+
return True
|
| 89 |
+
if _load_error is not None: # we already tried and failed
|
| 90 |
+
return False
|
| 91 |
+
try:
|
| 92 |
+
import torch
|
| 93 |
+
from transformers import AutoModelForImageTextToText, AutoProcessor
|
| 94 |
+
print(f"[vision] loading {VISION_MODEL_ID} (first call only)...")
|
| 95 |
+
_processor = AutoProcessor.from_pretrained(VISION_MODEL_ID,
|
| 96 |
+
trust_remote_code=True)
|
| 97 |
+
# Pick a placement strategy that AVOIDS disk offload. The "auto" map
|
| 98 |
+
# on a low-RAM machine offloads parts to disk, producing the
|
| 99 |
+
# "meta device" warning AND a shape-mismatch error at inference time
|
| 100 |
+
# (weights placed in the wrong submodules). We pick explicitly:
|
| 101 |
+
# - CUDA available β use it (ZeroGPU's H200, your dev box).
|
| 102 |
+
# - MPS available β Apple Silicon GPU (M1/M2/M3 Macs).
|
| 103 |
+
# - else CPU (slow but correct).
|
| 104 |
+
if torch.cuda.is_available():
|
| 105 |
+
target_device = "cuda"
|
| 106 |
+
dtype = torch.bfloat16
|
| 107 |
+
elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
|
| 108 |
+
target_device = "mps"
|
| 109 |
+
dtype = torch.float16 # mps doesn't love bfloat16
|
| 110 |
+
else:
|
| 111 |
+
target_device = "cpu"
|
| 112 |
+
dtype = torch.float32 # cpu can't do bfloat16 reliably
|
| 113 |
+
print(f"[vision] device={target_device} dtype={dtype}")
|
| 114 |
+
_model = AutoModelForImageTextToText.from_pretrained(
|
| 115 |
+
VISION_MODEL_ID,
|
| 116 |
+
torch_dtype=dtype,
|
| 117 |
+
trust_remote_code=True,
|
| 118 |
+
low_cpu_mem_usage=True,
|
| 119 |
+
).to(target_device)
|
| 120 |
+
_model.eval()
|
| 121 |
+
print("[vision] loaded.")
|
| 122 |
+
return True
|
| 123 |
+
except Exception as e:
|
| 124 |
+
_load_error = f"{type(e).__name__}: {e}"
|
| 125 |
+
print(f"[vision] load failed β falling back to stroke analysis. "
|
| 126 |
+
f"reason: {_load_error}")
|
| 127 |
+
traceback.print_exc()
|
| 128 |
+
return False
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def vision_available():
|
| 132 |
+
"""True iff the vision model is loaded (or can be lazily loaded).
|
| 133 |
+
Cheap to call; the actual model load is deferred to the first describe()."""
|
| 134 |
+
if _model is not None:
|
| 135 |
+
return True
|
| 136 |
+
if _load_error is not None:
|
| 137 |
+
return False
|
| 138 |
+
# We DON'T trigger a load here β that would block the UI. Just report
|
| 139 |
+
# whether transformers is importable, as a hint to the UI.
|
| 140 |
+
try:
|
| 141 |
+
import transformers # noqa: F401
|
| 142 |
+
import torch # noqa: F401
|
| 143 |
+
return True
|
| 144 |
+
except ImportError:
|
| 145 |
+
return False
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
# duration=120s gives us plenty of slack on ZeroGPU (a 1.3B vision pass at
|
| 149 |
+
# 16x downsample should take ~3-10s on H200, but first-call warmup can add
|
| 150 |
+
# time and we never want to bail mid-inference).
|
| 151 |
+
@spaces.GPU(duration=120)
|
| 152 |
+
def describe_with_vision(image, seed=None):
|
| 153 |
+
"""Run MiniCPM-V on the image. On ZeroGPU this is the only function that
|
| 154 |
+
holds a GPU. Returns the SAME shape as strokes.analyze_drawing()."""
|
| 155 |
+
if not _try_load():
|
| 156 |
+
# Model can't load β caller should fall back. Signal cleanly.
|
| 157 |
+
return None
|
| 158 |
+
|
| 159 |
+
import torch
|
| 160 |
+
from PIL import Image
|
| 161 |
+
import numpy as np
|
| 162 |
+
|
| 163 |
+
# ZeroGPU model: loaded on CPU at startup, moved to CUDA inside this
|
| 164 |
+
# decorated function for the duration of the call. Detect by checking if
|
| 165 |
+
# the model's current device differs from CUDA availability β if a GPU
|
| 166 |
+
# is now visible and the model is on CPU, move it.
|
| 167 |
+
global _model
|
| 168 |
+
if (torch.cuda.is_available()
|
| 169 |
+
and next(_model.parameters()).device.type == "cpu"):
|
| 170 |
+
print("[vision] ZeroGPU attached β moving model to CUDA for this call")
|
| 171 |
+
_model = _model.to("cuda")
|
| 172 |
+
|
| 173 |
+
pil = _to_pil(image)
|
| 174 |
+
if pil is None:
|
| 175 |
+
return None
|
| 176 |
+
|
| 177 |
+
# Build the chat messages in the format MiniCPM-V 4.6 expects.
|
| 178 |
+
messages = [{
|
| 179 |
+
"role": "user",
|
| 180 |
+
"content": [
|
| 181 |
+
{"type": "image", "image": pil},
|
| 182 |
+
{"type": "text", "text": DESCRIBE_PROMPT},
|
| 183 |
+
],
|
| 184 |
+
}]
|
| 185 |
+
|
| 186 |
+
try:
|
| 187 |
+
inputs = _processor.apply_chat_template(
|
| 188 |
+
messages,
|
| 189 |
+
add_generation_prompt=True,
|
| 190 |
+
tokenize=True,
|
| 191 |
+
return_dict=True,
|
| 192 |
+
return_tensors="pt",
|
| 193 |
+
# 16x = fewer visual tokens = faster. Right precision for our task.
|
| 194 |
+
downsample_mode="16x",
|
| 195 |
+
).to(_model.device, dtype=_model.dtype)
|
| 196 |
+
|
| 197 |
+
with torch.no_grad():
|
| 198 |
+
out_ids = _model.generate(
|
| 199 |
+
**inputs,
|
| 200 |
+
max_new_tokens=48,
|
| 201 |
+
do_sample=False, # deterministic β keeps it concise
|
| 202 |
+
)
|
| 203 |
+
# Strip the input prompt tokens from the front before decoding
|
| 204 |
+
prompt_len = inputs["input_ids"].shape[1]
|
| 205 |
+
text = _processor.batch_decode(out_ids[:, prompt_len:],
|
| 206 |
+
skip_special_tokens=True)[0].strip()
|
| 207 |
+
except Exception as e:
|
| 208 |
+
print(f"[vision] inference failed: {type(e).__name__}: {e}")
|
| 209 |
+
return None
|
| 210 |
+
|
| 211 |
+
text = _clean_description(text)
|
| 212 |
+
if not text:
|
| 213 |
+
return None
|
| 214 |
+
|
| 215 |
+
return {
|
| 216 |
+
"loves": text,
|
| 217 |
+
"mood": "from your drawing",
|
| 218 |
+
"summary": f'Your drawing looks like: "{text}"',
|
| 219 |
+
"source": "vision",
|
| 220 |
+
}
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def describe(image, seed=None, prefer="auto"):
|
| 224 |
+
"""High-level entry: prefers vision, falls back to strokes.
|
| 225 |
+
|
| 226 |
+
prefer:
|
| 227 |
+
"auto" β vision if available, else strokes.
|
| 228 |
+
"vision" β try vision; fall back to strokes only on failure.
|
| 229 |
+
"strokes"β always strokes (fast path, no GPU needed).
|
| 230 |
+
"""
|
| 231 |
+
from draw.strokes import analyze_drawing # local import keeps strokes optional
|
| 232 |
+
|
| 233 |
+
if prefer == "strokes" or not vision_available():
|
| 234 |
+
return _with_source(analyze_drawing(image, seed=seed), "strokes")
|
| 235 |
+
|
| 236 |
+
# Try vision, fall back silently on failure
|
| 237 |
+
result = describe_with_vision(image, seed=seed)
|
| 238 |
+
if result:
|
| 239 |
+
return result
|
| 240 |
+
return _with_source(analyze_drawing(image, seed=seed), "strokes")
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
# ---------------------------------------------------------------------------
|
| 244 |
+
# helpers
|
| 245 |
+
# ---------------------------------------------------------------------------
|
| 246 |
+
|
| 247 |
+
def _to_pil(image):
|
| 248 |
+
"""Accept PIL, numpy array, or Gradio ImageEditor dict; return PIL.Image
|
| 249 |
+
in RGB. Returns None for an effectively-empty canvas (so the caller can
|
| 250 |
+
fall back to a default rather than feeding noise to the model)."""
|
| 251 |
+
from PIL import Image
|
| 252 |
+
import numpy as np
|
| 253 |
+
|
| 254 |
+
if image is None:
|
| 255 |
+
return None
|
| 256 |
+
if isinstance(image, dict):
|
| 257 |
+
# ImageEditor/Sketchpad: prefer the merged composite, fall back to
|
| 258 |
+
# the image/background. Avoid `a or b or c` β for numpy arrays it
|
| 259 |
+
# raises "truth value is ambiguous". Test each key explicitly.
|
| 260 |
+
picked = None
|
| 261 |
+
for key in ("composite", "image", "background"):
|
| 262 |
+
v = image.get(key)
|
| 263 |
+
if v is not None:
|
| 264 |
+
picked = v
|
| 265 |
+
break
|
| 266 |
+
if picked is None and image.get("layers"):
|
| 267 |
+
picked = image["layers"][0]
|
| 268 |
+
image = picked
|
| 269 |
+
if image is None:
|
| 270 |
+
return None
|
| 271 |
+
if isinstance(image, np.ndarray):
|
| 272 |
+
if image.size == 0:
|
| 273 |
+
return None
|
| 274 |
+
if image.ndim == 2:
|
| 275 |
+
image = Image.fromarray(image).convert("RGB")
|
| 276 |
+
else:
|
| 277 |
+
image = Image.fromarray(image[..., :3].astype(np.uint8))
|
| 278 |
+
else:
|
| 279 |
+
image = image.convert("RGB")
|
| 280 |
+
|
| 281 |
+
# Sanity: if the canvas is almost-white, the model will return nonsense.
|
| 282 |
+
# Let the caller decide what to do β return None means "treat as empty".
|
| 283 |
+
arr = np.asarray(image)
|
| 284 |
+
if (arr > 240).all():
|
| 285 |
+
return None
|
| 286 |
+
|
| 287 |
+
# Cap size β MiniCPM-V handles big images but a 2048x2048 canvas is
|
| 288 |
+
# wasteful and slows inference. 768 on the long side is plenty for a
|
| 289 |
+
# child's drawing and matches the model's typical training resolution.
|
| 290 |
+
w, h = image.size
|
| 291 |
+
if max(w, h) > 768:
|
| 292 |
+
scale = 768 / max(w, h)
|
| 293 |
+
image = image.resize((int(w * scale), int(h * scale)), Image.LANCZOS)
|
| 294 |
+
return image
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
def _clean_description(text):
|
| 298 |
+
"""Strip artifacts MiniCPM-V sometimes adds, enforce length, and replace
|
| 299 |
+
drawing-technique words ('stick figure', 'sketch') with what a parent
|
| 300 |
+
would say ('a little person'). This is the WARMING layer β the prompt
|
| 301 |
+
stays literal (so the model doesn't hallucinate context that isn't on
|
| 302 |
+
the canvas), and this post-process softens any clinical jargon."""
|
| 303 |
+
import re
|
| 304 |
+
if not text:
|
| 305 |
+
return ""
|
| 306 |
+
# Drop everything after first newline / period if it's a long ramble
|
| 307 |
+
text = text.strip().strip('"').strip("'")
|
| 308 |
+
text = re.split(r"[\n\.]", text, maxsplit=1)[0].strip()
|
| 309 |
+
# Remove leading "I see " / "The image shows " etc β we want the bare list
|
| 310 |
+
text = re.sub(r"^(i see|the image shows|this drawing shows|i can see)\s+",
|
| 311 |
+
"", text, flags=re.IGNORECASE).strip()
|
| 312 |
+
|
| 313 |
+
# Substitute drawing-technique words with warm parent-language. The vision
|
| 314 |
+
# model is technically correct ("stick figure") but emotionally cold for
|
| 315 |
+
# a lullaby. We want "a little person" instead. Order matters β match
|
| 316 |
+
# specific phrases before generic ones; consume preceding articles so we
|
| 317 |
+
# don't end up with "a a little person".
|
| 318 |
+
JARGON_FIXES = [
|
| 319 |
+
(r"\b(?:a\s+|the\s+|one\s+)?stick[\s-]?figures?\b", "a little person"),
|
| 320 |
+
(r"\b(?:some\s+|the\s+)?stick[\s-]?people\b", "little people"),
|
| 321 |
+
(r"\b(?:simple|crude|rough)\s+drawings?\s+of\s+", ""),
|
| 322 |
+
(r"\b(?:a\s+)?sketch(?:es)?\s+of\s+", ""),
|
| 323 |
+
(r"\b(?:a\s+)?(?:line|pencil|crayon|colou?red)\s+drawings?\s+of\s+", ""),
|
| 324 |
+
(r"\bdoodles?\s+of\s+", ""),
|
| 325 |
+
(r"\bscribbles?\s+of\s+", ""),
|
| 326 |
+
(r"\b(?:simple|basic|crude|rough)\s+shapes?\b", "shapes"),
|
| 327 |
+
# If the cleaner left "a little person of dad, mom, baby" β replace
|
| 328 |
+
# the awkward "of <list of people>" with just the list of people.
|
| 329 |
+
(r"a little person of\s+", ""),
|
| 330 |
+
(r"little people of\s+", ""),
|
| 331 |
+
]
|
| 332 |
+
for pattern, replacement in JARGON_FIXES:
|
| 333 |
+
text = re.sub(pattern, replacement, text, flags=re.IGNORECASE)
|
| 334 |
+
# Collapse double spaces / spaces-before-commas / repeated articles
|
| 335 |
+
text = re.sub(r"\s+", " ", text).strip()
|
| 336 |
+
text = re.sub(r"\s+,", ",", text)
|
| 337 |
+
text = re.sub(r"\b(a|an|the)\s+\1\b", r"\1", text, flags=re.IGNORECASE)
|
| 338 |
+
|
| 339 |
+
# Hard length cap (the prompt asks for 8 words; clip to ~12 to be safe)
|
| 340 |
+
words = text.split()
|
| 341 |
+
if len(words) > 12:
|
| 342 |
+
text = " ".join(words[:12]).rstrip(",")
|
| 343 |
+
return text
|
| 344 |
+
|
| 345 |
+
|
| 346 |
+
def _with_source(result, source):
|
| 347 |
+
"""Tag a result dict with which engine produced it (for UI hints)."""
|
| 348 |
+
if result:
|
| 349 |
+
result = dict(result)
|
| 350 |
+
result["source"] = source
|
| 351 |
+
return result
|
requirements.txt
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Lolaby β drawing-first edition
|
| 2 |
+
# Tested with Gradio 5.x; works on Gradio 6.x with the small launch() change
|
| 3 |
+
# already made in app.py.
|
| 4 |
+
|
| 5 |
+
# ---- Web framework ----------------------------------------------------------
|
| 6 |
+
gradio>=5.0
|
| 7 |
+
|
| 8 |
+
# ---- Lyric generation (CPU, llama.cpp) --------------------------------------
|
| 9 |
+
# Stays on CPU even on ZeroGPU β the vision step is the only GPU consumer.
|
| 10 |
+
llama-cpp-python>=0.3.14
|
| 11 |
+
|
| 12 |
+
# ---- Vision (MiniCPM-V 4.6 via HF transformers, GPU on ZeroGPU) -------------
|
| 13 |
+
# Use a recent transformers β MiniCPM-V 4.6's native HF support landed in 4.45+.
|
| 14 |
+
transformers>=4.45
|
| 15 |
+
# Pillow for image handling on both the vision and stroke paths.
|
| 16 |
+
pillow>=10
|
| 17 |
+
# torchvision is needed by MiniCPM-V's image processor (torchvision backend).
|
| 18 |
+
# torch is required by transformers but ZeroGPU provides its own CUDA build
|
| 19 |
+
# at runtime, so we don't pin a CUDA-flavoured wheel here.
|
| 20 |
+
torch>=2.4
|
| 21 |
+
torchvision>=0.19
|
| 22 |
+
|
| 23 |
+
# ZeroGPU runtime helper β no-op outside HF Spaces, so safe to keep listed.
|
| 24 |
+
spaces
|
| 25 |
+
|
| 26 |
+
# ---- Audio + numerics -------------------------------------------------------
|
| 27 |
+
numpy<2 # several synths and llama-cpp interop want numpy 1.x
|
| 28 |
+
scipy>=1.10
|
| 29 |
+
soundfile>=0.12
|
| 30 |
+
|
| 31 |
+
# ---- TTS (Kokoro, the spoken lullaby voice) ---------------------------------
|
| 32 |
+
kokoro>=0.9.4
|
| 33 |
+
# kokoro pulls torch transitively but we pin it explicitly above.
|
synths/__init__.py
ADDED
|
File without changes
|
synths/guitar.py
ADDED
|
@@ -0,0 +1,393 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Acoustic guitar synthesizer β rebuilt from spectral analysis of two
|
| 3 |
+
Freesound acoustic guitar reference recordings.
|
| 4 |
+
|
| 5 |
+
Analysis findings (F#4 isolated note from ref_guitar2):
|
| 6 |
+
Spectrum (ΓN is harmonic of fundamental):
|
| 7 |
+
H1 (Γ1.0): amp 202 the fundamental
|
| 8 |
+
H2 (Γ2.0): amp 55 weaker than H1 and H3
|
| 9 |
+
H3 (Γ3.0): amp 235 STRONGER than fundamental β steel string signature
|
| 10 |
+
H4 (Γ4.0): amp 26
|
| 11 |
+
H5 (Γ5.0): amp 23
|
| 12 |
+
H6 (Γ6.0): amp 13
|
| 13 |
+
H7-H13: present out to 5kHz+ β bright, articulate
|
| 14 |
+
|
| 15 |
+
Envelope (post-attack peak):
|
| 16 |
+
Attack to peak: ~50-90ms
|
| 17 |
+
-3dB: 186ms
|
| 18 |
+
-6dB: 348ms
|
| 19 |
+
-12dB: ~700-1100ms
|
| 20 |
+
|
| 21 |
+
Strong pluck transient on attack (pick or fingernail click).
|
| 22 |
+
|
| 23 |
+
The dominant-H3 character is what makes a steel-string acoustic guitar
|
| 24 |
+
sound bright and "country" rather than warm and "classical". A naive
|
| 25 |
+
Karplus-Strong does NOT reproduce this β it generates a monotonically
|
| 26 |
+
falling harmonic series. So we layer KS (for natural string sympathetic
|
| 27 |
+
behavior) with explicit boosted-H3 partials and pluck transient noise.
|
| 28 |
+
|
| 29 |
+
Public interface (used by app.py):
|
| 30 |
+
GuitarSynth().sequence(events, effects=ACOUSTIC_PRESET) β np.ndarray
|
| 31 |
+
ACOUSTIC_PRESET: list of effect callables, applied in order
|
| 32 |
+
|
| 33 |
+
Event format:
|
| 34 |
+
{"type": "chord", "name": "C", "time": 0.0, "duration": 3.0,
|
| 35 |
+
"direction": "down", "spread_ms": 60,
|
| 36 |
+
"volume": 0.6, "decay": 0.997, "brightness": 0.5}
|
| 37 |
+
{"type": "note", "name": "G4", "time": 0.0, "duration": 1.0,
|
| 38 |
+
"volume": 1.0, "brightness": 0.5}
|
| 39 |
+
"""
|
| 40 |
+
|
| 41 |
+
import numpy as np
|
| 42 |
+
from scipy.signal import lfilter, butter
|
| 43 |
+
|
| 44 |
+
SR = 44100
|
| 45 |
+
|
| 46 |
+
NOTE_TO_SEMI = {
|
| 47 |
+
"C": 0, "C#": 1, "Db": 1, "D": 2, "D#": 3, "Eb": 3, "E": 4, "F": 5,
|
| 48 |
+
"F#": 6, "Gb": 6, "G": 7, "G#": 8, "Ab": 8, "A": 9, "A#": 10, "Bb": 10, "B": 11,
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
# Chord voicings (semitone offsets from root). Tuned for the guitar's
|
| 52 |
+
# natural register β fuller than a triad to sound like a strummed shape.
|
| 53 |
+
CHORD_INTERVALS = {
|
| 54 |
+
"": [0, 7, 12, 16, 19, 24], # major: root + 5 + oct + 3rd-up + 5-up + 2oct
|
| 55 |
+
"m": [0, 7, 12, 15, 19, 24], # minor: same but b3
|
| 56 |
+
"7": [0, 7, 10, 16, 19, 22],
|
| 57 |
+
"m7": [0, 7, 10, 15, 19, 22],
|
| 58 |
+
"maj7": [0, 7, 11, 16, 19, 23],
|
| 59 |
+
"sus2": [0, 7, 12, 14, 19, 24],
|
| 60 |
+
"sus4": [0, 7, 12, 17, 19, 24],
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def parse_chord(name):
|
| 65 |
+
name = name.strip()
|
| 66 |
+
if len(name) >= 2 and name[1] in ("#", "b"):
|
| 67 |
+
root, quality = name[:2], name[2:]
|
| 68 |
+
else:
|
| 69 |
+
root, quality = name[:1], name[1:]
|
| 70 |
+
return root, quality
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def note_to_hz(name):
|
| 74 |
+
name = name.strip()
|
| 75 |
+
for i, ch in enumerate(name):
|
| 76 |
+
if ch.isdigit() or ch == "-":
|
| 77 |
+
pitch = name[:i]
|
| 78 |
+
octave = int(name[i:])
|
| 79 |
+
break
|
| 80 |
+
else:
|
| 81 |
+
pitch, octave = name, 4
|
| 82 |
+
midi = 12 * (octave + 1) + NOTE_TO_SEMI[pitch]
|
| 83 |
+
return 440.0 * 2 ** ((midi - 69) / 12)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def chord_to_freqs(chord_name, base_octave=3):
|
| 87 |
+
"""Return list of frequencies in Hz for each string in the chord voicing."""
|
| 88 |
+
root, quality = parse_chord(chord_name)
|
| 89 |
+
intervals = CHORD_INTERVALS.get(quality, CHORD_INTERVALS[""])
|
| 90 |
+
root_midi = 12 * (base_octave + 1) + NOTE_TO_SEMI[root]
|
| 91 |
+
return [440.0 * 2 ** ((root_midi + i - 69) / 12) for i in intervals]
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def add_at(track, audio, start_sec, sr=SR):
|
| 95 |
+
start = int(start_sec * sr)
|
| 96 |
+
end = start + len(audio)
|
| 97 |
+
if end > len(track):
|
| 98 |
+
track = np.pad(track, (0, end - len(track)))
|
| 99 |
+
track[start:end] += audio
|
| 100 |
+
return track
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def normalize(x, target=0.95):
|
| 104 |
+
peak = float(np.max(np.abs(x)))
|
| 105 |
+
if peak > 1e-9:
|
| 106 |
+
return x * (target / peak)
|
| 107 |
+
return x
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
# ---------- effect building blocks ----------
|
| 111 |
+
|
| 112 |
+
def lowpass(audio, cutoff=7500, sr=SR, order=2):
|
| 113 |
+
b, a = butter(order, cutoff / (sr / 2), btype="low")
|
| 114 |
+
return lfilter(b, a, audio).astype(np.float32)
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def highpass(audio, cutoff=80, sr=SR, order=1):
|
| 118 |
+
b, a = butter(order, cutoff / (sr / 2), btype="high")
|
| 119 |
+
return lfilter(b, a, audio).astype(np.float32)
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def gentle_compress(audio, threshold=0.7, ratio=2.5):
|
| 123 |
+
"""Soft-knee compression for evening out plucks."""
|
| 124 |
+
out = audio.copy()
|
| 125 |
+
over = np.abs(out) > threshold
|
| 126 |
+
sign = np.sign(out[over])
|
| 127 |
+
excess = np.abs(out[over]) - threshold
|
| 128 |
+
out[over] = sign * (threshold + excess / ratio)
|
| 129 |
+
return out
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def body_resonance(audio, sr=SR, mix=0.18):
|
| 133 |
+
"""
|
| 134 |
+
Body resonance simulation β three narrow bandpass peaks at the
|
| 135 |
+
typical acoustic guitar body resonances:
|
| 136 |
+
- Helmholtz (air): ~100 Hz
|
| 137 |
+
- Top plate: ~200 Hz
|
| 138 |
+
- Back plate: ~380 Hz
|
| 139 |
+
Mixed back in at low gain β gives the "boxy warm body" sound.
|
| 140 |
+
"""
|
| 141 |
+
res = np.zeros_like(audio)
|
| 142 |
+
for freq, q_width, gain in [
|
| 143 |
+
(100, 60, 0.40),
|
| 144 |
+
(200, 100, 0.35),
|
| 145 |
+
(380, 150, 0.25),
|
| 146 |
+
]:
|
| 147 |
+
low = max(20, freq - q_width)
|
| 148 |
+
high = min(sr / 2 * 0.95, freq + q_width)
|
| 149 |
+
b, a = butter(2, [low / (sr / 2), high / (sr / 2)], btype="band")
|
| 150 |
+
res += lfilter(b, a, audio).astype(np.float32) * gain
|
| 151 |
+
return audio + res * mix
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
# ACOUSTIC_PRESET: pipeline applied to the final mix. Each callable takes
|
| 155 |
+
# (audio,) and returns audio.
|
| 156 |
+
ACOUSTIC_PRESET = [
|
| 157 |
+
lambda x: body_resonance(x, mix=0.20),
|
| 158 |
+
lambda x: highpass(x, cutoff=70),
|
| 159 |
+
lambda x: lowpass(x, cutoff=8500),
|
| 160 |
+
lambda x: gentle_compress(x, threshold=0.7, ratio=2.5),
|
| 161 |
+
lambda x: normalize(x, target=0.92),
|
| 162 |
+
]
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
# ---------- core string synthesis ----------
|
| 166 |
+
|
| 167 |
+
def karplus_strong(freq, n_samples, sr=SR, damping=0.997, brightness=0.5):
|
| 168 |
+
"""
|
| 169 |
+
Karplus-Strong plucked string. Provides the natural harmonic comb +
|
| 170 |
+
realistic decay-with-pitch behavior (high frequencies decay faster).
|
| 171 |
+
"""
|
| 172 |
+
delay_len = max(2, int(round(sr / freq)))
|
| 173 |
+
|
| 174 |
+
# Pluck excitation β pre-filtered noise. Brightness controls the
|
| 175 |
+
# high-frequency content of the initial energy.
|
| 176 |
+
excitation = (np.random.rand(delay_len).astype(np.float32) * 2 - 1)
|
| 177 |
+
cutoff = 600 + brightness * 5000
|
| 178 |
+
b, a = butter(2, cutoff / (sr / 2), btype="low")
|
| 179 |
+
excitation = lfilter(b, a, excitation).astype(np.float32)
|
| 180 |
+
|
| 181 |
+
buf = excitation.copy()
|
| 182 |
+
out = np.zeros(n_samples, dtype=np.float32)
|
| 183 |
+
prev = 0.0
|
| 184 |
+
i = 0
|
| 185 |
+
for s in range(n_samples):
|
| 186 |
+
cur = buf[i]
|
| 187 |
+
out[s] = cur
|
| 188 |
+
new = damping * 0.5 * (cur + prev)
|
| 189 |
+
buf[i] = new
|
| 190 |
+
prev = cur
|
| 191 |
+
i = (i + 1) % delay_len
|
| 192 |
+
return out
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def pluck_transient(sr=SR, brightness=0.5, hardness=0.5, length_ms=8):
|
| 196 |
+
"""
|
| 197 |
+
Brief filtered noise burst β the pick/fingernail attack click.
|
| 198 |
+
"""
|
| 199 |
+
n = int(length_ms / 1000 * sr)
|
| 200 |
+
if n <= 0:
|
| 201 |
+
return np.zeros(0, dtype=np.float32)
|
| 202 |
+
noise = np.random.randn(n).astype(np.float32) * (0.10 + hardness * 0.10)
|
| 203 |
+
# Bandpass for "pick" character β 1.5-6 kHz
|
| 204 |
+
low = 1200 + hardness * 600
|
| 205 |
+
high = min(sr / 2 * 0.95, 4000 + brightness * 2500)
|
| 206 |
+
b, a = butter(2, [low / (sr / 2), high / (sr / 2)], btype="band")
|
| 207 |
+
click = lfilter(b, a, noise).astype(np.float32)
|
| 208 |
+
# Fast exponential decay
|
| 209 |
+
env = np.exp(-np.arange(n) / (sr * 0.0015))
|
| 210 |
+
return (click * env).astype(np.float32)
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
class GuitarSynth:
|
| 214 |
+
"""
|
| 215 |
+
Acoustic steel-string guitar synthesizer.
|
| 216 |
+
|
| 217 |
+
Hybrid: Karplus-Strong (sympathetic behavior + natural harmonic comb)
|
| 218 |
+
layered with explicit boosted-H3 partials (the steel-string signature)
|
| 219 |
+
and a pluck transient.
|
| 220 |
+
"""
|
| 221 |
+
|
| 222 |
+
def __init__(self, sr=SR):
|
| 223 |
+
self.sr = sr
|
| 224 |
+
|
| 225 |
+
def note(self, freq, duration_s, volume=1.0, brightness=0.5):
|
| 226 |
+
"""
|
| 227 |
+
Single plucked string.
|
| 228 |
+
|
| 229 |
+
brightness: 0..1. 0.5 matches the reference reasonably; bump higher
|
| 230 |
+
for more bright country-strum feel.
|
| 231 |
+
"""
|
| 232 |
+
sr = self.sr
|
| 233 |
+
# Render long enough for the full decay tail (~1.5s perceptible)
|
| 234 |
+
n = max(int(duration_s * sr), int(1.8 * sr))
|
| 235 |
+
t = np.arange(n) / sr
|
| 236 |
+
|
| 237 |
+
# Damping scales with pitch: lower notes sustain longer.
|
| 238 |
+
# Tuned so F#4 (370 Hz) hits the ~350ms -6dB measured in the reference.
|
| 239 |
+
damping = max(0.9955, 0.9992 - (freq / 18000.0))
|
| 240 |
+
ks = karplus_strong(freq, n, sr, damping=damping, brightness=brightness)
|
| 241 |
+
|
| 242 |
+
# Envelope: ~70ms attack-to-peak (the reference had peak at 70-93ms,
|
| 243 |
+
# which is the body response building up), then exponential decay
|
| 244 |
+
# with tau β 0.9s (matches -6dB at ~348ms after a 60ms plateau).
|
| 245 |
+
plateau_n = int(0.06 * sr)
|
| 246 |
+
decay_tau = 0.9
|
| 247 |
+
env = np.ones(n, dtype=np.float32)
|
| 248 |
+
if plateau_n > 1:
|
| 249 |
+
env[:plateau_n] = np.linspace(0.6, 1.0, plateau_n) ** 0.8
|
| 250 |
+
decay_n = n - plateau_n
|
| 251 |
+
decay_t = np.arange(decay_n) / sr
|
| 252 |
+
env[plateau_n:] = np.exp(-decay_t / decay_tau)
|
| 253 |
+
ks = ks * env
|
| 254 |
+
|
| 255 |
+
# Explicit partials to inject the steel-string-signature H3 dominance.
|
| 256 |
+
# Reference ratios H1:H2:H3:H4:H5 β 1.00 : 0.27 : 1.16 : 0.13 : 0.11.
|
| 257 |
+
# KS already provides H1 + falling tail. We add JUST enough H3 to push
|
| 258 |
+
# it slightly above the fundamental, plus a touch of H5 for sparkle.
|
| 259 |
+
h3_amp = 0.07 + brightness * 0.04
|
| 260 |
+
h5_amp = 0.02 + brightness * 0.02
|
| 261 |
+
|
| 262 |
+
h3_env_tau = decay_tau * 0.7
|
| 263 |
+
h5_env_tau = decay_tau * 0.5
|
| 264 |
+
h3_env = np.ones(n, dtype=np.float32)
|
| 265 |
+
h5_env = np.ones(n, dtype=np.float32)
|
| 266 |
+
if plateau_n > 1:
|
| 267 |
+
h3_env[:plateau_n] = np.linspace(0.6, 1.0, plateau_n) ** 0.8
|
| 268 |
+
h5_env[:plateau_n] = np.linspace(0.6, 1.0, plateau_n) ** 0.8
|
| 269 |
+
h3_env[plateau_n:] = np.exp(-decay_t / h3_env_tau)
|
| 270 |
+
h5_env[plateau_n:] = np.exp(-decay_t / h5_env_tau)
|
| 271 |
+
|
| 272 |
+
partials = h3_amp * np.sin(2 * np.pi * freq * 3 * t) * h3_env
|
| 273 |
+
if freq * 5 < sr / 2 * 0.9:
|
| 274 |
+
partials += h5_amp * np.sin(2 * np.pi * freq * 5 * t) * h5_env
|
| 275 |
+
|
| 276 |
+
# Pluck transient β adds the attack "click"
|
| 277 |
+
click = pluck_transient(sr=sr, brightness=brightness,
|
| 278 |
+
hardness=0.3 + brightness * 0.2, length_ms=6)
|
| 279 |
+
|
| 280 |
+
signal = ks * 0.75 + partials * 0.5
|
| 281 |
+
# Stamp the click at the front
|
| 282 |
+
click_n = len(click)
|
| 283 |
+
if click_n > 0 and click_n < n:
|
| 284 |
+
signal[:click_n] += click * 0.5
|
| 285 |
+
|
| 286 |
+
# Soft attack ramp (3ms) to prevent any DC pop
|
| 287 |
+
ramp_n = int(0.003 * sr)
|
| 288 |
+
if ramp_n > 1:
|
| 289 |
+
signal[:ramp_n] *= np.linspace(0, 1, ramp_n)
|
| 290 |
+
# Release tail
|
| 291 |
+
rel_n = int(0.05 * sr)
|
| 292 |
+
if rel_n > 1:
|
| 293 |
+
signal[-rel_n:] *= np.linspace(1, 0, rel_n)
|
| 294 |
+
|
| 295 |
+
# Final lowpass to tame any KS aliasing or harshness
|
| 296 |
+
signal = lowpass(signal, cutoff=9000, sr=sr)
|
| 297 |
+
|
| 298 |
+
# Normalize per-note for consistent loudness across pitches
|
| 299 |
+
peak = float(np.max(np.abs(signal)))
|
| 300 |
+
if peak > 1e-9:
|
| 301 |
+
signal = signal / peak * 0.85
|
| 302 |
+
return (signal * volume).astype(np.float32)
|
| 303 |
+
|
| 304 |
+
def chord(self, chord_name, duration_s, base_octave=3,
|
| 305 |
+
direction="down", spread_ms=20,
|
| 306 |
+
volume=0.7, brightness=0.5, decay=None):
|
| 307 |
+
"""
|
| 308 |
+
Render a strummed chord.
|
| 309 |
+
|
| 310 |
+
spread_ms is the TOTAL time the pick takes to cross all strings
|
| 311 |
+
(not per-string). A real strum sweeps in 15-30ms; anything over
|
| 312 |
+
~80ms starts to sound arpeggiated rather than strummed.
|
| 313 |
+
|
| 314 |
+
direction: "down" = bass-to-treble (typical down-strum)
|
| 315 |
+
"up" = treble-to-bass (up-strum)
|
| 316 |
+
"""
|
| 317 |
+
freqs = chord_to_freqs(chord_name, base_octave=base_octave)
|
| 318 |
+
if direction == "up":
|
| 319 |
+
freqs = list(reversed(freqs))
|
| 320 |
+
|
| 321 |
+
# Total strum across all strings β per-string stagger
|
| 322 |
+
# Subtract 1 because we have N-1 gaps between N strings
|
| 323 |
+
n_strings = len(freqs)
|
| 324 |
+
if n_strings <= 1:
|
| 325 |
+
stagger = 0.0
|
| 326 |
+
else:
|
| 327 |
+
stagger = (spread_ms / 1000.0) / (n_strings - 1)
|
| 328 |
+
|
| 329 |
+
note_dur = max(duration_s, 1.5)
|
| 330 |
+
total_len = int((duration_s + stagger * n_strings + 1.5) * self.sr)
|
| 331 |
+
out = np.zeros(total_len, dtype=np.float32)
|
| 332 |
+
|
| 333 |
+
for i, freq in enumerate(freqs):
|
| 334 |
+
# Voicing: bass string slightly stronger
|
| 335 |
+
voice_vol = 1.0 if i == 0 else (0.85 if i < 3 else 0.7)
|
| 336 |
+
note_audio = self.note(freq, note_dur,
|
| 337 |
+
volume=volume * voice_vol,
|
| 338 |
+
brightness=brightness)
|
| 339 |
+
start = int(i * stagger * self.sr)
|
| 340 |
+
end = start + len(note_audio)
|
| 341 |
+
if end > len(out):
|
| 342 |
+
out = np.pad(out, (0, end - len(out)))
|
| 343 |
+
out[start:end] += note_audio
|
| 344 |
+
|
| 345 |
+
# Normalize the strummed chord
|
| 346 |
+
peak = float(np.max(np.abs(out)))
|
| 347 |
+
if peak > 1.0:
|
| 348 |
+
out = out / peak
|
| 349 |
+
return out.astype(np.float32)
|
| 350 |
+
|
| 351 |
+
def sequence(self, events, effects=None):
|
| 352 |
+
"""
|
| 353 |
+
Render a list of chord/note events. Apply `effects` pipeline to the
|
| 354 |
+
final mix (in order). ACOUSTIC_PRESET is the recommended default.
|
| 355 |
+
"""
|
| 356 |
+
if not events:
|
| 357 |
+
return np.zeros(int(self.sr), dtype=np.float32)
|
| 358 |
+
|
| 359 |
+
events = sorted(events, key=lambda e: e["time"])
|
| 360 |
+
end_time = max(e["time"] + max(e["duration"], 0.6) for e in events) + 2.0
|
| 361 |
+
track = np.zeros(int(end_time * self.sr) + 1, dtype=np.float32)
|
| 362 |
+
|
| 363 |
+
for ev in events:
|
| 364 |
+
typ = ev.get("type")
|
| 365 |
+
if typ == "chord":
|
| 366 |
+
# Accept the same params app.py passes today: name, time,
|
| 367 |
+
# duration, direction, spread_ms, volume, decay, brightness.
|
| 368 |
+
# `decay` is informational β actual decay is per-pitch.
|
| 369 |
+
audio = self.chord(
|
| 370 |
+
ev["name"], ev["duration"],
|
| 371 |
+
base_octave=ev.get("octave", ev.get("base_octave", 3)),
|
| 372 |
+
direction=ev.get("direction", "down"),
|
| 373 |
+
spread_ms=ev.get("spread_ms", 60),
|
| 374 |
+
volume=ev.get("volume", 0.7),
|
| 375 |
+
brightness=ev.get("brightness", 0.5),
|
| 376 |
+
)
|
| 377 |
+
elif typ == "note":
|
| 378 |
+
freq = note_to_hz(ev["name"])
|
| 379 |
+
audio = self.note(
|
| 380 |
+
freq, max(ev["duration"], 1.5),
|
| 381 |
+
volume=ev.get("volume", 1.0),
|
| 382 |
+
brightness=ev.get("brightness", 0.5),
|
| 383 |
+
)
|
| 384 |
+
else:
|
| 385 |
+
continue
|
| 386 |
+
track = add_at(track, audio, ev["time"], sr=self.sr)
|
| 387 |
+
|
| 388 |
+
# Apply post-effects pipeline if given
|
| 389 |
+
if effects:
|
| 390 |
+
for fx in effects:
|
| 391 |
+
track = fx(track)
|
| 392 |
+
|
| 393 |
+
return track
|
synths/harp.py
ADDED
|
@@ -0,0 +1,381 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Harp synthesizer β based on spectral analysis of two Freesound reference clips
|
| 3 |
+
(freesound_community-harp-motif2 and remembrance-harp).
|
| 4 |
+
|
| 5 |
+
Analysis findings:
|
| 6 |
+
- Strong fundamental, with H2 at ~0.67Γ the fundamental's amplitude
|
| 7 |
+
(warm "woody" character β much more H2 than the xylophone's 1:0.03)
|
| 8 |
+
- H3 weak (~0.06Γ), faint H6 visible
|
| 9 |
+
- Envelope: ~70ms plateau at peak, then exponential decay
|
| 10 |
+
-6dB at ~210ms post-peak
|
| 11 |
+
-20dB at ~770ms
|
| 12 |
+
-40dB at ~1.7s
|
| 13 |
+
Fully inaudible by ~2-3s
|
| 14 |
+
- Chord arpeggio interval (when arpeggiated): ~110-200ms between strikes
|
| 15 |
+
|
| 16 |
+
Hybrid synthesis: a long-decay Karplus-Strong string for the natural harmonic
|
| 17 |
+
comb and sympathetic-ringing character, layered with explicit sine partials
|
| 18 |
+
for the fundamental + H2 to match the reference's warmth.
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
import numpy as np
|
| 22 |
+
from scipy.signal import lfilter, butter
|
| 23 |
+
|
| 24 |
+
SR = 44100
|
| 25 |
+
|
| 26 |
+
NOTE = {
|
| 27 |
+
"C": 0, "C#": 1, "Db": 1, "D": 2, "D#": 3, "Eb": 3, "E": 4, "F": 5,
|
| 28 |
+
"F#": 6, "Gb": 6, "G": 7, "G#": 8, "Ab": 8, "A": 9, "A#": 10, "Bb": 10, "B": 11,
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
CHORD_INTERVALS = {
|
| 32 |
+
"": [0, 4, 7, 12],
|
| 33 |
+
"m": [0, 3, 7, 12],
|
| 34 |
+
"7": [0, 4, 7, 10],
|
| 35 |
+
"m7": [0, 3, 7, 10],
|
| 36 |
+
"maj7": [0, 4, 7, 11],
|
| 37 |
+
"sus2": [0, 2, 7, 12],
|
| 38 |
+
"sus4": [0, 5, 7, 12],
|
| 39 |
+
}
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def parse_chord(name):
|
| 43 |
+
name = name.strip()
|
| 44 |
+
if len(name) >= 2 and name[1] in ("#", "b"):
|
| 45 |
+
root, quality = name[:2], name[2:]
|
| 46 |
+
else:
|
| 47 |
+
root, quality = name[:1], name[1:]
|
| 48 |
+
return root, quality
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def note_to_hz(name):
|
| 52 |
+
name = name.strip()
|
| 53 |
+
for i, ch in enumerate(name):
|
| 54 |
+
if ch.isdigit() or ch == "-":
|
| 55 |
+
pitch = name[:i]
|
| 56 |
+
octave = int(name[i:])
|
| 57 |
+
break
|
| 58 |
+
else:
|
| 59 |
+
pitch, octave = name, 4
|
| 60 |
+
midi = 12 * (octave + 1) + NOTE[pitch]
|
| 61 |
+
return 440.0 * 2 ** ((midi - 69) / 12)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def chord_to_freqs(chord_name, octave=3):
|
| 65 |
+
root, quality = parse_chord(chord_name)
|
| 66 |
+
intervals = CHORD_INTERVALS.get(quality, CHORD_INTERVALS[""])
|
| 67 |
+
root_midi = 12 * (octave + 1) + NOTE[root]
|
| 68 |
+
return [440.0 * 2 ** ((root_midi + i - 69) / 12) for i in intervals]
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def add_at(track, audio, start_sec):
|
| 72 |
+
start = int(start_sec * SR)
|
| 73 |
+
end = start + len(audio)
|
| 74 |
+
if end > len(track):
|
| 75 |
+
track = np.pad(track, (0, end - len(track)))
|
| 76 |
+
track[start:end] += audio
|
| 77 |
+
return track
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def simple_reverb(audio, sr, room_size=0.6, damping=0.55, wet=0.22):
|
| 81 |
+
"""
|
| 82 |
+
Multi-tap delay reverb. Produces ~1.5-2s tail with diffusion.
|
| 83 |
+
|
| 84 |
+
The tail is built from 6 delayed, lowpassed, feedback-recirculated copies
|
| 85 |
+
of the input. Each tap has a slightly different delay (prime-ish ratios
|
| 86 |
+
to avoid metallic resonance) and decay factor.
|
| 87 |
+
|
| 88 |
+
Parameters:
|
| 89 |
+
room_size: 0..1 β scales tap delays (bigger = longer tail)
|
| 90 |
+
damping: 0..1 β lowpass cutoff scaling (more = darker tail)
|
| 91 |
+
wet: 0..1 β wet/dry mix (0 = dry, 1 = all reverb)
|
| 92 |
+
"""
|
| 93 |
+
if wet <= 0.0:
|
| 94 |
+
return audio
|
| 95 |
+
|
| 96 |
+
n = len(audio)
|
| 97 |
+
# Tap delays in ms (prime-ratio spread, scaled by room_size)
|
| 98 |
+
base_delays_ms = [29.0, 37.0, 53.0, 67.0, 89.0, 113.0]
|
| 99 |
+
# Per-tap gain β earlier taps stronger
|
| 100 |
+
gains = [0.55, 0.48, 0.42, 0.36, 0.30, 0.24]
|
| 101 |
+
|
| 102 |
+
# Build the wet signal
|
| 103 |
+
wet_buf = np.zeros(n + int(sr * 2.0), dtype=np.float32)
|
| 104 |
+
|
| 105 |
+
# Lowpass for the diffuse tail (darker as `damping` increases)
|
| 106 |
+
cutoff = max(800.0, 5000.0 * (1.0 - damping))
|
| 107 |
+
b_lp, a_lp = butter(2, cutoff / (sr / 2), btype="low")
|
| 108 |
+
pre_lp = lfilter(b_lp, a_lp, audio).astype(np.float32)
|
| 109 |
+
|
| 110 |
+
for ms, g in zip(base_delays_ms, gains):
|
| 111 |
+
delay_samples = int(ms * sr / 1000.0 * (0.6 + room_size * 0.8))
|
| 112 |
+
end = delay_samples + n
|
| 113 |
+
if end > len(wet_buf):
|
| 114 |
+
wet_buf = np.pad(wet_buf, (0, end - len(wet_buf)))
|
| 115 |
+
wet_buf[delay_samples:end] += pre_lp * g
|
| 116 |
+
|
| 117 |
+
# Feedback recirculation β pass the wet signal through a single combed
|
| 118 |
+
# delay to build a smooth tail. Decay set by `room_size`.
|
| 119 |
+
fb_delay = int(0.071 * sr * (0.6 + room_size * 0.8))
|
| 120 |
+
fb_gain = 0.45 + room_size * 0.20 # 0.45..0.65
|
| 121 |
+
for k in range(1, 5):
|
| 122 |
+
offset = fb_delay * k
|
| 123 |
+
if offset >= len(wet_buf):
|
| 124 |
+
break
|
| 125 |
+
wet_buf[offset:] += wet_buf[:-offset] * (fb_gain ** k) * 0.5
|
| 126 |
+
|
| 127 |
+
# Trim wet to length, lowpass again to smooth
|
| 128 |
+
wet_buf = wet_buf[:n]
|
| 129 |
+
wet_buf = lfilter(b_lp, a_lp, wet_buf).astype(np.float32)
|
| 130 |
+
|
| 131 |
+
return (audio * (1.0 - wet) + wet_buf * wet).astype(np.float32)
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def _karplus_strong(freq, n_samples, sr, damping=0.9985):
|
| 135 |
+
"""
|
| 136 |
+
Karplus-Strong plucked string. Integer-sample delay line.
|
| 137 |
+
|
| 138 |
+
Note on tuning: at very high pitches (above A6 or so), the integer-sample
|
| 139 |
+
delay rounding introduces a small pitch error (a few cents). We accept
|
| 140 |
+
this because (a) musical content rarely sits above A6 in lullabies, and
|
| 141 |
+
(b) the perceived "out of tune" issue in chord contexts isn't this β it's
|
| 142 |
+
overlapping ring-out from previous chords. We mitigate that by fading the
|
| 143 |
+
track between chord changes in sequence().
|
| 144 |
+
|
| 145 |
+
Damping near 1.0 = long sustain. For the harp, ~0.9985 (very slow decay).
|
| 146 |
+
"""
|
| 147 |
+
delay_len = max(2, int(round(sr / freq)))
|
| 148 |
+
# Soft pre-filtered noise excitation β finger pad not pick
|
| 149 |
+
excitation = (np.random.rand(delay_len).astype(np.float32) * 2 - 1)
|
| 150 |
+
b, a = butter(2, [200 / (sr / 2), 3500 / (sr / 2)], btype="band")
|
| 151 |
+
excitation = lfilter(b, a, excitation).astype(np.float32)
|
| 152 |
+
|
| 153 |
+
buf = excitation.copy()
|
| 154 |
+
out = np.zeros(n_samples, dtype=np.float32)
|
| 155 |
+
prev = 0.0
|
| 156 |
+
i = 0
|
| 157 |
+
for s in range(n_samples):
|
| 158 |
+
cur = buf[i]
|
| 159 |
+
out[s] = cur
|
| 160 |
+
# One-pole lowpass averaging = string damping
|
| 161 |
+
new = damping * 0.5 * (cur + prev)
|
| 162 |
+
buf[i] = new
|
| 163 |
+
prev = cur
|
| 164 |
+
i = (i + 1) % delay_len
|
| 165 |
+
return out
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
class HarpSynth:
|
| 169 |
+
"""Concert / Celtic harp β long-sustain plucked string with warm H2."""
|
| 170 |
+
|
| 171 |
+
def __init__(self, sr=SR):
|
| 172 |
+
self.sr = sr
|
| 173 |
+
|
| 174 |
+
def note(self, freq, duration_s, volume=1.0, brightness=0.4):
|
| 175 |
+
"""
|
| 176 |
+
One plucked harp string.
|
| 177 |
+
|
| 178 |
+
brightness: 0..1 β controls additive H2/H3 level and pluck attack
|
| 179 |
+
noise. 0.4 matches the reference well.
|
| 180 |
+
"""
|
| 181 |
+
sr = self.sr
|
| 182 |
+
# Render long enough for the full decay tail β at least 2.5s.
|
| 183 |
+
# Reference decays to -40dB at ~1.7s and is inaudible by ~2.5s.
|
| 184 |
+
n = max(int(duration_s * sr), int(2.5 * sr))
|
| 185 |
+
t = np.arange(n) / sr
|
| 186 |
+
|
| 187 |
+
# Damping: scaled with pitch. Low strings sustain longer (concert harp
|
| 188 |
+
# bass strings ring ~3s; high strings ~1-1.5s).
|
| 189 |
+
damping = max(0.994, 0.9991 - (freq / 8000.0))
|
| 190 |
+
ks = _karplus_strong(freq, n, sr, damping=damping)
|
| 191 |
+
|
| 192 |
+
# Reference envelope: ~70ms plateau then exponential decay with tau
|
| 193 |
+
# such that -6dB @ 210ms, -20dB @ 770ms. tau β 0.33s.
|
| 194 |
+
plateau_end = 0.07
|
| 195 |
+
decay_tau = 0.33
|
| 196 |
+
env = np.ones(n, dtype=np.float32)
|
| 197 |
+
plateau_n = int(plateau_end * sr)
|
| 198 |
+
if plateau_n > 1:
|
| 199 |
+
env[:plateau_n] = np.linspace(0.85, 1.0, plateau_n)
|
| 200 |
+
decay_n = n - plateau_n
|
| 201 |
+
decay_t = np.arange(decay_n) / sr
|
| 202 |
+
env[plateau_n:] = np.exp(-decay_t / decay_tau)
|
| 203 |
+
|
| 204 |
+
ks = ks * env
|
| 205 |
+
|
| 206 |
+
# Additive partials to match reference spectrum:
|
| 207 |
+
# fundamental:H2 β 1:0.67 H3 β 1:0.06
|
| 208 |
+
# KS naturally produces some H2 but typically under-weights it for
|
| 209 |
+
# a warm harp tone. Layer explicit sines with their own envelopes.
|
| 210 |
+
h1_amp = 0.45
|
| 211 |
+
h2_amp = 0.30 + brightness * 0.10 # the warm "woody" component
|
| 212 |
+
h3_amp = 0.04 + brightness * 0.025
|
| 213 |
+
|
| 214 |
+
# Partials use the same envelope curve but with slightly faster decay
|
| 215 |
+
# for higher harmonics (real strings lose high frequencies first)
|
| 216 |
+
h2_env_tau = decay_tau * 0.75
|
| 217 |
+
h3_env_tau = decay_tau * 0.55
|
| 218 |
+
h2_env = np.ones(n, dtype=np.float32)
|
| 219 |
+
h3_env = np.ones(n, dtype=np.float32)
|
| 220 |
+
if plateau_n > 1:
|
| 221 |
+
h2_env[:plateau_n] = np.linspace(0.85, 1.0, plateau_n)
|
| 222 |
+
h3_env[:plateau_n] = np.linspace(0.85, 1.0, plateau_n)
|
| 223 |
+
h2_env[plateau_n:] = np.exp(-decay_t / h2_env_tau)
|
| 224 |
+
h3_env[plateau_n:] = np.exp(-decay_t / h3_env_tau)
|
| 225 |
+
|
| 226 |
+
partials = (h1_amp * np.sin(2 * np.pi * freq * t) * env
|
| 227 |
+
+ h2_amp * np.sin(2 * np.pi * freq * 2 * t) * h2_env
|
| 228 |
+
+ (h3_amp * np.sin(2 * np.pi * freq * 3 * t) * h3_env
|
| 229 |
+
if freq * 3 < sr / 2 * 0.9 else 0))
|
| 230 |
+
|
| 231 |
+
signal = ks * 0.55 + partials
|
| 232 |
+
|
| 233 |
+
# Soft attack ramp β no clicks
|
| 234 |
+
atk_n = int(0.004 * sr)
|
| 235 |
+
if atk_n > 1:
|
| 236 |
+
signal[:atk_n] *= np.linspace(0, 1, atk_n)
|
| 237 |
+
|
| 238 |
+
# Gentle release tail
|
| 239 |
+
rel_n = int(0.05 * sr)
|
| 240 |
+
if rel_n > 1:
|
| 241 |
+
signal[-rel_n:] *= np.linspace(1, 0, rel_n)
|
| 242 |
+
|
| 243 |
+
# Final lowpass for warm tone β kills any KS high-freq harshness
|
| 244 |
+
b_warm, a_warm = butter(2, 4000 / (sr / 2), btype="low")
|
| 245 |
+
signal = lfilter(b_warm, a_warm, signal).astype(np.float32)
|
| 246 |
+
|
| 247 |
+
# Normalize per-note for consistent loudness across pitches
|
| 248 |
+
peak = float(np.max(np.abs(signal)))
|
| 249 |
+
if peak > 1e-9:
|
| 250 |
+
signal = signal / peak * 0.85
|
| 251 |
+
return (signal * volume).astype(np.float32)
|
| 252 |
+
|
| 253 |
+
def chord(self, chord_name, duration_s, octave=3,
|
| 254 |
+
volume=0.7, brightness=0.4,
|
| 255 |
+
arpeggio_ms=130, direction="up",
|
| 256 |
+
sympathetic=True):
|
| 257 |
+
"""
|
| 258 |
+
Rolled chord arpeggio β the harp signature.
|
| 259 |
+
|
| 260 |
+
arpeggio_ms: 80-180ms between successive note onsets. 130ms is dreamy.
|
| 261 |
+
sympathetic: add faint lowpassed delayed copy for ringing-strings feel.
|
| 262 |
+
"""
|
| 263 |
+
freqs = chord_to_freqs(chord_name, octave=octave)
|
| 264 |
+
if direction == "down":
|
| 265 |
+
freqs = list(reversed(freqs))
|
| 266 |
+
|
| 267 |
+
stagger = arpeggio_ms / 1000.0
|
| 268 |
+
note_dur = max(duration_s, 2.5)
|
| 269 |
+
total_len = int((duration_s + stagger * len(freqs) + 3.0) * self.sr)
|
| 270 |
+
out = np.zeros(total_len, dtype=np.float32)
|
| 271 |
+
|
| 272 |
+
for i, freq in enumerate(freqs):
|
| 273 |
+
# Voicing: bass slightly stronger, top slightly softer
|
| 274 |
+
if i == 0:
|
| 275 |
+
voice_vol = 1.0
|
| 276 |
+
elif i == len(freqs) - 1:
|
| 277 |
+
voice_vol = 0.7
|
| 278 |
+
else:
|
| 279 |
+
voice_vol = 0.82
|
| 280 |
+
note_audio = self.note(freq, note_dur,
|
| 281 |
+
volume=volume * voice_vol,
|
| 282 |
+
brightness=brightness)
|
| 283 |
+
start = int(i * stagger * self.sr)
|
| 284 |
+
end = start + len(note_audio)
|
| 285 |
+
if end > len(out):
|
| 286 |
+
out = np.pad(out, (0, end - len(out)))
|
| 287 |
+
out[start:end] += note_audio
|
| 288 |
+
|
| 289 |
+
# Sympathetic resonance: delayed lowpassed copy mixed back at low gain
|
| 290 |
+
if sympathetic:
|
| 291 |
+
delay_samples = int(0.025 * self.sr)
|
| 292 |
+
b_sym, a_sym = butter(2, 1200 / (self.sr / 2), btype="low")
|
| 293 |
+
tail = lfilter(b_sym, a_sym, out).astype(np.float32)
|
| 294 |
+
shifted = np.zeros_like(out)
|
| 295 |
+
shifted[delay_samples:] = tail[:-delay_samples] * 0.18
|
| 296 |
+
out = out + shifted
|
| 297 |
+
|
| 298 |
+
# NOTE: hall-style reverb is applied in sequence() at the mix level
|
| 299 |
+
# (one pass over the whole performance) rather than per-chord, so we
|
| 300 |
+
# don't double-process.
|
| 301 |
+
|
| 302 |
+
# Normalize the rolled chord
|
| 303 |
+
peak = float(np.max(np.abs(out)))
|
| 304 |
+
if peak > 1.0:
|
| 305 |
+
out = out / peak
|
| 306 |
+
return out.astype(np.float32)
|
| 307 |
+
|
| 308 |
+
def sequence(self, events):
|
| 309 |
+
"""Render chord and/or note events."""
|
| 310 |
+
if not events:
|
| 311 |
+
return np.zeros(int(self.sr), dtype=np.float32)
|
| 312 |
+
|
| 313 |
+
events = sorted(events, key=lambda e: e["time"])
|
| 314 |
+
end_time = max(e["time"] + max(e["duration"], 0.6) for e in events) + 3.0
|
| 315 |
+
track = np.zeros(int(end_time * self.sr) + 1, dtype=np.float32)
|
| 316 |
+
|
| 317 |
+
# Render each event into its own buffer, then mix with chord-aware
|
| 318 |
+
# tail damping so previous chords' rings don't beat against the new
|
| 319 |
+
# chord's harmonics (the main perceived "out of tune" sound).
|
| 320 |
+
chord_events = [e for e in events if e.get("type") == "chord"]
|
| 321 |
+
chord_starts = [e["time"] for e in chord_events]
|
| 322 |
+
|
| 323 |
+
for ev in events:
|
| 324 |
+
typ = ev.get("type")
|
| 325 |
+
if typ == "chord":
|
| 326 |
+
audio = self.chord(
|
| 327 |
+
ev["name"], ev["duration"],
|
| 328 |
+
octave=ev.get("octave", 3),
|
| 329 |
+
volume=ev.get("volume", 0.7),
|
| 330 |
+
brightness=ev.get("brightness", 0.4),
|
| 331 |
+
arpeggio_ms=ev.get("arpeggio_ms", 130),
|
| 332 |
+
direction=ev.get("direction", "up"),
|
| 333 |
+
sympathetic=ev.get("sympathetic", True),
|
| 334 |
+
)
|
| 335 |
+
|
| 336 |
+
# Find the next chord boundary AFTER this one. Apply a
|
| 337 |
+
# quick exponential fade-down at that boundary on this
|
| 338 |
+
# chord's audio, so its harmonics stop ringing into the
|
| 339 |
+
# next chord. 250ms fade window β short enough to feel like
|
| 340 |
+
# natural string-damping (a harpist's palm muting), long
|
| 341 |
+
# enough not to click.
|
| 342 |
+
this_t = ev["time"]
|
| 343 |
+
next_chord_starts = [t for t in chord_starts if t > this_t + 0.1]
|
| 344 |
+
if next_chord_starts:
|
| 345 |
+
next_t = min(next_chord_starts)
|
| 346 |
+
# Where in `audio` does the next chord fall?
|
| 347 |
+
relative_next = next_t - this_t # seconds
|
| 348 |
+
fade_start_sample = int(relative_next * self.sr)
|
| 349 |
+
if 0 < fade_start_sample < len(audio):
|
| 350 |
+
# Exponential fade from 1.0 β 0.18 over 250ms,
|
| 351 |
+
# then hold at 0.18 (don't kill the tail entirely β
|
| 352 |
+
# we want a hint of overhang for naturalness)
|
| 353 |
+
fade_n = int(0.25 * self.sr)
|
| 354 |
+
end_sample = min(fade_start_sample + fade_n, len(audio))
|
| 355 |
+
n = end_sample - fade_start_sample
|
| 356 |
+
if n > 0:
|
| 357 |
+
curve = np.linspace(0, 1, n) ** 1.6
|
| 358 |
+
envelope = 1.0 - curve * (1.0 - 0.18)
|
| 359 |
+
audio[fade_start_sample:end_sample] = (
|
| 360 |
+
audio[fade_start_sample:end_sample] * envelope
|
| 361 |
+
)
|
| 362 |
+
# Beyond the fade window, hold at 0.18
|
| 363 |
+
if end_sample < len(audio):
|
| 364 |
+
audio[end_sample:] *= 0.18
|
| 365 |
+
|
| 366 |
+
elif typ == "note":
|
| 367 |
+
freq = note_to_hz(ev["name"])
|
| 368 |
+
audio = self.note(
|
| 369 |
+
freq, max(ev["duration"], 2.0),
|
| 370 |
+
volume=ev.get("volume", 1.0),
|
| 371 |
+
brightness=ev.get("brightness", 0.4),
|
| 372 |
+
)
|
| 373 |
+
else:
|
| 374 |
+
continue
|
| 375 |
+
track = add_at(track, audio, ev["time"])
|
| 376 |
+
|
| 377 |
+
# Apply hall-style reverb to the whole performance. A single pass
|
| 378 |
+
# over the mix is cheaper and more coherent than per-event reverb.
|
| 379 |
+
track = simple_reverb(track, self.sr,
|
| 380 |
+
room_size=0.55, damping=0.50, wet=0.22)
|
| 381 |
+
return track
|
synths/musicbox.py
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Music box synthesizer β the quintessential lullaby instrument.
|
| 3 |
+
|
| 4 |
+
A music box uses steel tines plucked by pins on a rotating cylinder.
|
| 5 |
+
Acoustically, that produces a sound with very specific properties:
|
| 6 |
+
|
| 7 |
+
1) Near-pure sine fundamental (tines vibrate dominantly in their
|
| 8 |
+
fundamental mode, much more so than a struck-bar like xylophone)
|
| 9 |
+
2) A whisper of 2nd harmonic β slight metallic flavor
|
| 10 |
+
3) Brief inharmonic 'ping' at ~5-6Γ the fundamental β the stiffness
|
| 11 |
+
of the steel tine producing a non-integer-multiple partial that
|
| 12 |
+
fades fast (the metallic shimmer at attack)
|
| 13 |
+
4) Sharp, brief metallic attack transient β the pin-on-tine pluck.
|
| 14 |
+
Much sharper than a felt mallet, much more metallic than a piano
|
| 15 |
+
hammer. Brief HF burst (4-8 kHz) then immediate decay
|
| 16 |
+
5) Long slow decay (~2-3s for low tines, ~0.8-1.2s for high)
|
| 17 |
+
6) Distinctive 'bell' quality from the wooden box body that the tines
|
| 18 |
+
are mounted on β gives a slight reverb tail with low-mid emphasis
|
| 19 |
+
7) Sweet spot is the C5-C7 register; lower notes lose definition because
|
| 20 |
+
real music-box tines get physically large and unwieldy
|
| 21 |
+
|
| 22 |
+
Reverb is baked in (small wooden-box character, not a hall). This matters:
|
| 23 |
+
without the box body, the tines would sound thin and toy-like.
|
| 24 |
+
|
| 25 |
+
Public interface (mirrors other synths):
|
| 26 |
+
|
| 27 |
+
from synths.musicbox import MusicBoxSynth, SR
|
| 28 |
+
synth = MusicBoxSynth()
|
| 29 |
+
synth.sequence(events) β np.ndarray @ SR
|
| 30 |
+
|
| 31 |
+
Event format:
|
| 32 |
+
{"type": "chord", "name": "C", "time": 0.0, "duration": 3.0,
|
| 33 |
+
"octave": 5, "volume": 0.7, "spread_ms": 20, "direction": "up"}
|
| 34 |
+
{"type": "note", "name": "G5", "time": 0.0, "duration": 1.0,
|
| 35 |
+
"volume": 0.9}
|
| 36 |
+
"""
|
| 37 |
+
|
| 38 |
+
import numpy as np
|
| 39 |
+
from scipy.signal import butter, lfilter
|
| 40 |
+
|
| 41 |
+
SR = 44100
|
| 42 |
+
|
| 43 |
+
NOTE = {
|
| 44 |
+
"C": 0, "C#": 1, "Db": 1, "D": 2, "D#": 3, "Eb": 3, "E": 4, "F": 5,
|
| 45 |
+
"F#": 6, "Gb": 6, "G": 7, "G#": 8, "Ab": 8, "A": 9, "A#": 10, "Bb": 10, "B": 11,
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
CHORD_INTERVALS = {
|
| 49 |
+
"": [0, 4, 7, 12],
|
| 50 |
+
"m": [0, 3, 7, 12],
|
| 51 |
+
"7": [0, 4, 7, 10],
|
| 52 |
+
"m7": [0, 3, 7, 10],
|
| 53 |
+
"maj7": [0, 4, 7, 11],
|
| 54 |
+
"sus2": [0, 2, 7, 12],
|
| 55 |
+
"sus4": [0, 5, 7, 12],
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def parse_chord(name):
|
| 60 |
+
name = name.strip()
|
| 61 |
+
if len(name) >= 2 and name[1] in ("#", "b"):
|
| 62 |
+
root, quality = name[:2], name[2:]
|
| 63 |
+
else:
|
| 64 |
+
root, quality = name[:1], name[1:]
|
| 65 |
+
return root, quality
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def note_to_hz(name):
|
| 69 |
+
"""e.g. 'A5' β 880.0"""
|
| 70 |
+
name = name.strip()
|
| 71 |
+
for i, ch in enumerate(name):
|
| 72 |
+
if ch.isdigit() or ch == "-":
|
| 73 |
+
pitch = name[:i]
|
| 74 |
+
octave = int(name[i:])
|
| 75 |
+
break
|
| 76 |
+
else:
|
| 77 |
+
pitch, octave = name, 5
|
| 78 |
+
midi = 12 * (octave + 1) + NOTE[pitch]
|
| 79 |
+
return 440.0 * 2 ** ((midi - 69) / 12)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def chord_to_freqs(chord_name, octave=5):
|
| 83 |
+
root, quality = parse_chord(chord_name)
|
| 84 |
+
intervals = CHORD_INTERVALS.get(quality, CHORD_INTERVALS[""])
|
| 85 |
+
root_midi = 12 * (octave + 1) + NOTE[root]
|
| 86 |
+
return [440.0 * 2 ** ((root_midi + i - 69) / 12) for i in intervals]
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def add_at(track, audio, start_sec, sr=SR):
|
| 90 |
+
start = int(start_sec * sr)
|
| 91 |
+
end = start + len(audio)
|
| 92 |
+
if end > len(track):
|
| 93 |
+
track = np.pad(track, (0, end - len(track)))
|
| 94 |
+
track[start:end] += audio
|
| 95 |
+
return track
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def box_reverb(audio, sr=SR, wet=0.20):
|
| 99 |
+
"""
|
| 100 |
+
Small wooden-box reverb β the resonance of the music box's housing.
|
| 101 |
+
Short (~400ms tail), dark, intimate β NOT a concert hall.
|
| 102 |
+
|
| 103 |
+
Implementation: 4 lowpassed delay taps + brief feedback. Cheap.
|
| 104 |
+
"""
|
| 105 |
+
if wet <= 0.0:
|
| 106 |
+
return audio
|
| 107 |
+
n = len(audio)
|
| 108 |
+
# Tight tap spread β small box, not a room
|
| 109 |
+
delay_ms = [11.0, 19.0, 29.0, 41.0]
|
| 110 |
+
gains = [0.55, 0.42, 0.32, 0.24]
|
| 111 |
+
|
| 112 |
+
# Pre-lowpass (the wooden box doesn't reflect high frequencies well)
|
| 113 |
+
b_lp, a_lp = butter(2, 2200 / (sr / 2), btype="low")
|
| 114 |
+
pre = lfilter(b_lp, a_lp, audio).astype(np.float32)
|
| 115 |
+
|
| 116 |
+
wet_buf = np.zeros(n + int(sr * 0.5), dtype=np.float32)
|
| 117 |
+
for ms, g in zip(delay_ms, gains):
|
| 118 |
+
d = int(ms * sr / 1000.0)
|
| 119 |
+
end = d + n
|
| 120 |
+
if end > len(wet_buf):
|
| 121 |
+
wet_buf = np.pad(wet_buf, (0, end - len(wet_buf)))
|
| 122 |
+
wet_buf[d:end] += pre * g
|
| 123 |
+
|
| 124 |
+
# Feedback for smooth decay
|
| 125 |
+
fb_delay = int(0.053 * sr)
|
| 126 |
+
for k in range(1, 3):
|
| 127 |
+
offset = fb_delay * k
|
| 128 |
+
if offset >= len(wet_buf):
|
| 129 |
+
break
|
| 130 |
+
wet_buf[offset:] += wet_buf[:-offset] * (0.35 ** k) * 0.5
|
| 131 |
+
|
| 132 |
+
wet_buf = wet_buf[:n]
|
| 133 |
+
wet_buf = lfilter(b_lp, a_lp, wet_buf).astype(np.float32)
|
| 134 |
+
|
| 135 |
+
return (audio * (1.0 - wet) + wet_buf * wet).astype(np.float32)
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def _pluck_transient(freq, sr=SR, length_ms=22):
|
| 139 |
+
"""
|
| 140 |
+
The sharp metallic pin-on-tine pluck.
|
| 141 |
+
|
| 142 |
+
Two components:
|
| 143 |
+
(a) Brief broadband click in 2-8kHz (the pin striking the tine edge β
|
| 144 |
+
this is the 'tink' sound)
|
| 145 |
+
(b) A short transient sine burst at ~5Γ the fundamental (the inharmonic
|
| 146 |
+
stiffness partial of the tine β gives the metallic shimmer)
|
| 147 |
+
|
| 148 |
+
Decays in ~10ms. Much sharper than a felt mallet or piano hammer.
|
| 149 |
+
"""
|
| 150 |
+
n = int(length_ms / 1000 * sr)
|
| 151 |
+
if n <= 0:
|
| 152 |
+
return np.zeros(0, dtype=np.float32)
|
| 153 |
+
t = np.arange(n) / sr
|
| 154 |
+
|
| 155 |
+
# Component A: HF click
|
| 156 |
+
noise = np.random.randn(n).astype(np.float32)
|
| 157 |
+
low = 2000.0
|
| 158 |
+
high = min(sr / 2 * 0.95, 8000.0)
|
| 159 |
+
b, a = butter(2, [low / (sr / 2), high / (sr / 2)], btype="band")
|
| 160 |
+
click = lfilter(b, a, noise).astype(np.float32)
|
| 161 |
+
# Very fast decay β 2ms time constant
|
| 162 |
+
click_env = np.exp(-np.arange(n) / (sr * 0.002))
|
| 163 |
+
click = click * click_env * 0.18
|
| 164 |
+
|
| 165 |
+
# Component B: brief inharmonic stiffness partial (~5Γ fundamental)
|
| 166 |
+
# This is what gives music box its distinctive sweet metallic ping.
|
| 167 |
+
# Higher tines have a stronger one (smaller tines are more stiff-coupled).
|
| 168 |
+
if freq < 2000: # above this it'd alias or be inaudible
|
| 169 |
+
stiff_freq = freq * 5.4 # slightly inharmonic (not exact 5Γ)
|
| 170 |
+
if stiff_freq < sr / 2 * 0.9:
|
| 171 |
+
stiff_env = np.exp(-np.arange(n) / (sr * 0.008)) # 8ms decay
|
| 172 |
+
stiff_amp = 0.12 + min(0.10, freq / 8000)
|
| 173 |
+
stiff = stiff_amp * np.sin(2 * np.pi * stiff_freq * t) * stiff_env
|
| 174 |
+
click += stiff.astype(np.float32)
|
| 175 |
+
|
| 176 |
+
return click.astype(np.float32)
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
class MusicBoxSynth:
|
| 180 |
+
"""
|
| 181 |
+
Music box: steel tines plucked by cylinder pins.
|
| 182 |
+
Near-pure sine + faint H2 + brief inharmonic ping + long slow decay,
|
| 183 |
+
finished with a small wooden-box reverb.
|
| 184 |
+
"""
|
| 185 |
+
|
| 186 |
+
def __init__(self, sr=SR):
|
| 187 |
+
self.sr = sr
|
| 188 |
+
|
| 189 |
+
def note(self, freq, duration_s, volume=1.0):
|
| 190 |
+
"""
|
| 191 |
+
One plucked tine.
|
| 192 |
+
|
| 193 |
+
duration_s is mostly ignored beyond a minimum β the tine has its own
|
| 194 |
+
natural decay (~1-2.5s depending on pitch). You can't damp a music
|
| 195 |
+
box tine mid-note.
|
| 196 |
+
"""
|
| 197 |
+
sr = self.sr
|
| 198 |
+
# Bass tines sustain ~2.5s; treble ~0.8s.
|
| 199 |
+
# Use pitch-scaled tau: tau = 0.6 + (110/freq) * 0.5
|
| 200 |
+
# β at 110Hz: 1.1s tau (long); at 880Hz: 0.66s tau
|
| 201 |
+
tau = max(0.30, min(1.20, 0.6 + (110.0 / freq) * 0.5))
|
| 202 |
+
|
| 203 |
+
# Render to fully capture the decay tail (5Ο β -40dB)
|
| 204 |
+
n = max(int(duration_s * sr), int(min(3.0, tau * 5) * sr))
|
| 205 |
+
t = np.arange(n) / sr
|
| 206 |
+
|
| 207 |
+
# Fundamental β exponential decay, no plateau (tines don't plateau
|
| 208 |
+
# like soft-mallet bars do, they start decaying immediately).
|
| 209 |
+
fund_env = np.exp(-t / tau)
|
| 210 |
+
signal = (np.sin(2 * np.pi * freq * t) * fund_env).astype(np.float32)
|
| 211 |
+
|
| 212 |
+
# Whisper of 2nd harmonic β characteristic metallic flavor
|
| 213 |
+
# H2 amp ~0.06Γ fundamental, decays faster (tau Γ 0.5)
|
| 214 |
+
if freq * 2 < sr / 2 * 0.95:
|
| 215 |
+
h2_env = np.exp(-t / (tau * 0.5))
|
| 216 |
+
signal += (0.06 * np.sin(2 * np.pi * freq * 2 * t)
|
| 217 |
+
* h2_env).astype(np.float32)
|
| 218 |
+
|
| 219 |
+
# Tiny H3 for very low notes only β adds warmth in the bass register
|
| 220 |
+
if freq < 200 and freq * 3 < sr / 2 * 0.9:
|
| 221 |
+
h3_env = np.exp(-t / (tau * 0.4))
|
| 222 |
+
signal += (0.025 * np.sin(2 * np.pi * freq * 3 * t)
|
| 223 |
+
* h3_env).astype(np.float32)
|
| 224 |
+
|
| 225 |
+
# Stamp the pluck transient at the front
|
| 226 |
+
pluck = _pluck_transient(freq, sr=sr)
|
| 227 |
+
if len(pluck) > 0 and len(pluck) < n:
|
| 228 |
+
signal[:len(pluck)] += pluck
|
| 229 |
+
|
| 230 |
+
# Tiny attack ramp on the sine body (no DC click)
|
| 231 |
+
ramp_n = int(0.0015 * sr)
|
| 232 |
+
if ramp_n > 1:
|
| 233 |
+
signal[:ramp_n] *= np.linspace(0, 1, ramp_n)
|
| 234 |
+
|
| 235 |
+
# Normalize per-note then scale by volume
|
| 236 |
+
peak = float(np.max(np.abs(signal)))
|
| 237 |
+
if peak > 1e-9:
|
| 238 |
+
signal = signal / peak * 0.85
|
| 239 |
+
return (signal * volume).astype(np.float32)
|
| 240 |
+
|
| 241 |
+
def chord(self, chord_name, duration_s, octave=5,
|
| 242 |
+
volume=0.7, spread_ms=20, direction="up"):
|
| 243 |
+
"""
|
| 244 |
+
Render a chord. Real music boxes either pluck all tines simultaneously
|
| 245 |
+
(the cylinder pins for a chord line up) or sequentially (the cylinder
|
| 246 |
+
rotates through the notes). Both are valid.
|
| 247 |
+
|
| 248 |
+
spread_ms 0-40 = simultaneous chord; 100+ = sequential cylinder roll.
|
| 249 |
+
"""
|
| 250 |
+
freqs = chord_to_freqs(chord_name, octave=octave)
|
| 251 |
+
if direction == "down":
|
| 252 |
+
freqs = list(reversed(freqs))
|
| 253 |
+
|
| 254 |
+
n_keys = len(freqs)
|
| 255 |
+
stagger = (spread_ms / 1000.0) / max(1, n_keys - 1) if n_keys > 1 else 0
|
| 256 |
+
|
| 257 |
+
note_dur = max(duration_s, 1.5)
|
| 258 |
+
total_len = int((duration_s + stagger * n_keys + 3.0) * self.sr)
|
| 259 |
+
out = np.zeros(total_len, dtype=np.float32)
|
| 260 |
+
|
| 261 |
+
for i, freq in enumerate(freqs):
|
| 262 |
+
# Voicing: top voice slightly stronger (the melody-bearing tine
|
| 263 |
+
# in a real music box is often the prominent one). Bass slightly
|
| 264 |
+
# softer because low tines can rumble.
|
| 265 |
+
if i == n_keys - 1:
|
| 266 |
+
voice_vol = 1.0
|
| 267 |
+
elif i == 0:
|
| 268 |
+
voice_vol = 0.78
|
| 269 |
+
else:
|
| 270 |
+
voice_vol = 0.85
|
| 271 |
+
note_audio = self.note(freq, note_dur, volume=volume * voice_vol)
|
| 272 |
+
start = int(i * stagger * self.sr)
|
| 273 |
+
end = start + len(note_audio)
|
| 274 |
+
if end > len(out):
|
| 275 |
+
out = np.pad(out, (0, end - len(out)))
|
| 276 |
+
out[start:end] += note_audio
|
| 277 |
+
|
| 278 |
+
peak = float(np.max(np.abs(out)))
|
| 279 |
+
if peak > 1.0:
|
| 280 |
+
out = out / peak
|
| 281 |
+
return out.astype(np.float32)
|
| 282 |
+
|
| 283 |
+
def sequence(self, events):
|
| 284 |
+
"""Render mixed chord and note events. Applies the wooden-box reverb
|
| 285 |
+
once at the mix level."""
|
| 286 |
+
if not events:
|
| 287 |
+
return np.zeros(int(self.sr), dtype=np.float32)
|
| 288 |
+
|
| 289 |
+
events = sorted(events, key=lambda e: e["time"])
|
| 290 |
+
end_time = max(e["time"] + max(e["duration"], 0.6) for e in events) + 2.0
|
| 291 |
+
track = np.zeros(int(end_time * self.sr) + 1, dtype=np.float32)
|
| 292 |
+
|
| 293 |
+
for ev in events:
|
| 294 |
+
typ = ev.get("type")
|
| 295 |
+
if typ == "chord":
|
| 296 |
+
audio = self.chord(
|
| 297 |
+
ev["name"], ev["duration"],
|
| 298 |
+
octave=ev.get("octave", 5),
|
| 299 |
+
volume=ev.get("volume", 0.7),
|
| 300 |
+
spread_ms=ev.get("spread_ms", 20),
|
| 301 |
+
direction=ev.get("direction", "up"),
|
| 302 |
+
)
|
| 303 |
+
elif typ == "note":
|
| 304 |
+
freq = note_to_hz(ev["name"])
|
| 305 |
+
audio = self.note(
|
| 306 |
+
freq, ev["duration"],
|
| 307 |
+
volume=ev.get("volume", 1.0),
|
| 308 |
+
)
|
| 309 |
+
else:
|
| 310 |
+
continue
|
| 311 |
+
track = add_at(track, audio, ev["time"], sr=self.sr)
|
| 312 |
+
|
| 313 |
+
# Apply the wooden-box reverb at the mix level.
|
| 314 |
+
track = box_reverb(track, self.sr, wet=0.22)
|
| 315 |
+
|
| 316 |
+
# Safety normalize so overlapping rings don't clip.
|
| 317 |
+
peak = float(np.max(np.abs(track)))
|
| 318 |
+
if peak > 0.95:
|
| 319 |
+
track = track * (0.95 / peak)
|
| 320 |
+
return track
|
synths/ocarina.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Whistle synthesizer β human-like whistling.
|
| 3 |
+
|
| 4 |
+
A whistle is acoustically simple: near-sine fundamental + small 2nd harmonic,
|
| 5 |
+
vibrato that fades in, soft envelope, light breath noise, smooth portamento
|
| 6 |
+
between notes.
|
| 7 |
+
|
| 8 |
+
Use as the MELODY layer over a rhythm instrument (guitar/piano/flute).
|
| 9 |
+
|
| 10 |
+
Example:
|
| 11 |
+
from whistle import WhistleSynth
|
| 12 |
+
synth = WhistleSynth()
|
| 13 |
+
events = [
|
| 14 |
+
{"type": "note", "name": "G5", "time": 0.0, "duration": 1.0},
|
| 15 |
+
{"type": "note", "name": "A5", "time": 1.0, "duration": 0.5},
|
| 16 |
+
{"type": "note", "name": "G5", "time": 1.5, "duration": 1.5},
|
| 17 |
+
]
|
| 18 |
+
audio = synth.sequence(events)
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
import numpy as np
|
| 22 |
+
from scipy.signal import butter, lfilter
|
| 23 |
+
|
| 24 |
+
SR = 44100
|
| 25 |
+
|
| 26 |
+
NOTE = {
|
| 27 |
+
"C": 0, "C#": 1, "Db": 1, "D": 2, "D#": 3, "Eb": 3, "E": 4, "F": 5,
|
| 28 |
+
"F#": 6, "Gb": 6, "G": 7, "G#": 8, "Ab": 8, "A": 9, "A#": 10, "Bb": 10, "B": 11,
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def note_to_hz(name):
|
| 33 |
+
"""e.g. 'A4' β 440.0"""
|
| 34 |
+
name = name.strip()
|
| 35 |
+
# Find where the digit starts
|
| 36 |
+
for i, ch in enumerate(name):
|
| 37 |
+
if ch.isdigit() or ch == "-":
|
| 38 |
+
pitch = name[:i]
|
| 39 |
+
octave = int(name[i:])
|
| 40 |
+
break
|
| 41 |
+
else:
|
| 42 |
+
pitch, octave = name, 4
|
| 43 |
+
midi = 12 * (octave + 1) + NOTE[pitch]
|
| 44 |
+
return 440.0 * 2 ** ((midi - 69) / 12)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def add_at(track, audio, start_sec):
|
| 48 |
+
start = int(start_sec * SR)
|
| 49 |
+
end = start + len(audio)
|
| 50 |
+
if end > len(track):
|
| 51 |
+
track = np.pad(track, (0, end - len(track)))
|
| 52 |
+
track[start:end] += audio
|
| 53 |
+
return track
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
class WhistleSynth:
|
| 57 |
+
"""Synthesizes a melodic whistle from note events."""
|
| 58 |
+
|
| 59 |
+
def __init__(self, sr=SR):
|
| 60 |
+
self.sr = sr
|
| 61 |
+
|
| 62 |
+
def note(self, freq, duration_s,
|
| 63 |
+
vibrato_hz=4.8, vibrato_cents=32,
|
| 64 |
+
attack_ms=120, release_ms=400,
|
| 65 |
+
breath_level=0.018,
|
| 66 |
+
prev_freq=None, glide_ms=80,
|
| 67 |
+
volume=1.0):
|
| 68 |
+
"""One whistled note."""
|
| 69 |
+
sr = self.sr
|
| 70 |
+
n = int(duration_s * sr)
|
| 71 |
+
if n <= 0:
|
| 72 |
+
return np.zeros(0, dtype=np.float32)
|
| 73 |
+
t = np.arange(n) / sr
|
| 74 |
+
|
| 75 |
+
# Pitch curve with optional glide from previous note
|
| 76 |
+
if prev_freq is not None and glide_ms > 0:
|
| 77 |
+
glide_n = min(int(glide_ms / 1000 * sr), n // 3)
|
| 78 |
+
freq_curve = np.full(n, freq, dtype=np.float32)
|
| 79 |
+
if glide_n > 1:
|
| 80 |
+
freq_curve[:glide_n] = np.linspace(prev_freq, freq, glide_n)
|
| 81 |
+
else:
|
| 82 |
+
freq_curve = np.full(n, freq, dtype=np.float32)
|
| 83 |
+
|
| 84 |
+
# Vibrato fades in (whistlers don't start with wobble)
|
| 85 |
+
vibrato_env = np.minimum((t - 0.2) / 0.3, 1.0).clip(0, 1)
|
| 86 |
+
depth_hz = freq_curve * (2 ** (vibrato_cents / 1200) - 1)
|
| 87 |
+
vibrato = depth_hz * vibrato_env * np.sin(2 * np.pi * vibrato_hz * t)
|
| 88 |
+
inst_freq = freq_curve + vibrato
|
| 89 |
+
|
| 90 |
+
# Phase from instantaneous frequency
|
| 91 |
+
phase = 2 * np.pi * np.cumsum(inst_freq) / sr
|
| 92 |
+
|
| 93 |
+
# Tone: fundamental + soft 2nd harmonic
|
| 94 |
+
tone = np.sin(phase) + 0.06 * np.sin(2 * phase)
|
| 95 |
+
|
| 96 |
+
# Breath: bandpassed noise around the fundamental
|
| 97 |
+
noise = np.random.randn(n).astype(np.float32)
|
| 98 |
+
f_center = float(np.mean(freq_curve))
|
| 99 |
+
low = max(80.0, f_center * 0.8)
|
| 100 |
+
high = min(sr / 2 * 0.95, f_center * 1.6)
|
| 101 |
+
b, a = butter(2, [low / (sr / 2), high / (sr / 2)], btype="band")
|
| 102 |
+
breath = lfilter(b, a, noise).astype(np.float32) * breath_level
|
| 103 |
+
|
| 104 |
+
out = (tone + breath).astype(np.float32)
|
| 105 |
+
|
| 106 |
+
# Envelope
|
| 107 |
+
env = np.ones(n, dtype=np.float32)
|
| 108 |
+
atk_n = min(int(attack_ms / 1000 * sr), n // 3)
|
| 109 |
+
rel_n = min(int(release_ms / 1000 * sr), n // 2)
|
| 110 |
+
if atk_n > 1:
|
| 111 |
+
env[:atk_n] = np.linspace(0, 1, atk_n) ** 1.5
|
| 112 |
+
if rel_n > 1:
|
| 113 |
+
env[-rel_n:] = np.linspace(1, 0, rel_n) ** 1.5
|
| 114 |
+
|
| 115 |
+
return (out * env * volume).astype(np.float32)
|
| 116 |
+
|
| 117 |
+
def _explode_chord(self, ev):
|
| 118 |
+
"""
|
| 119 |
+
Break a chord event into a slow broken-chord pattern of note events.
|
| 120 |
+
|
| 121 |
+
Ocarina is monophonic, so a rhythm-role chord is rendered as a slow
|
| 122 |
+
ascending arpeggio that takes up the chord duration.
|
| 123 |
+
"""
|
| 124 |
+
name = ev["name"]
|
| 125 |
+
# Parse chord
|
| 126 |
+
if len(name) >= 2 and name[1] in ("#", "b"):
|
| 127 |
+
root, quality = name[:2], name[2:]
|
| 128 |
+
else:
|
| 129 |
+
root, quality = name[:1], name[1:]
|
| 130 |
+
is_minor = "m" in quality and "maj" not in quality
|
| 131 |
+
intervals = [0, (3 if is_minor else 4), 7] # root, third, fifth
|
| 132 |
+
|
| 133 |
+
octave = ev.get("octave", 5)
|
| 134 |
+
root_midi = 12 * (octave + 1) + NOTE[root]
|
| 135 |
+
names = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"]
|
| 136 |
+
|
| 137 |
+
duration = ev["duration"]
|
| 138 |
+
t0 = ev["time"]
|
| 139 |
+
vol = ev.get("volume", 0.7)
|
| 140 |
+
# Three notes per chord, spread across most of the chord duration
|
| 141 |
+
note_dur = duration / 2.5
|
| 142 |
+
sub_events = []
|
| 143 |
+
for i, semi in enumerate(intervals):
|
| 144 |
+
midi = root_midi + semi
|
| 145 |
+
o = midi // 12 - 1
|
| 146 |
+
pitch_name = names[midi % 12]
|
| 147 |
+
sub_events.append({
|
| 148 |
+
"type": "note",
|
| 149 |
+
"name": f"{pitch_name}{o}",
|
| 150 |
+
"time": t0 + i * (duration / 3.5),
|
| 151 |
+
"duration": note_dur,
|
| 152 |
+
"volume": vol * (0.85 if i > 0 else 1.0),
|
| 153 |
+
})
|
| 154 |
+
return sub_events
|
| 155 |
+
|
| 156 |
+
def sequence(self, events):
|
| 157 |
+
"""
|
| 158 |
+
Render note and chord events. Chords become slow broken-chord
|
| 159 |
+
arpeggios (ocarina is monophonic).
|
| 160 |
+
"""
|
| 161 |
+
if not events:
|
| 162 |
+
return np.zeros(int(self.sr), dtype=np.float32)
|
| 163 |
+
|
| 164 |
+
# Expand chord events into note events
|
| 165 |
+
expanded = []
|
| 166 |
+
for ev in events:
|
| 167 |
+
if ev.get("type") == "chord":
|
| 168 |
+
expanded.extend(self._explode_chord(ev))
|
| 169 |
+
elif ev.get("type", "note") == "note":
|
| 170 |
+
expanded.append(ev)
|
| 171 |
+
|
| 172 |
+
events = sorted(expanded, key=lambda e: e["time"])
|
| 173 |
+
end_time = max(e["time"] + e["duration"] for e in events) + 0.5
|
| 174 |
+
track = np.zeros(int(end_time * self.sr) + 1, dtype=np.float32)
|
| 175 |
+
|
| 176 |
+
prev_freq = None
|
| 177 |
+
prev_end = -10.0
|
| 178 |
+
|
| 179 |
+
for ev in events:
|
| 180 |
+
freq = note_to_hz(ev["name"])
|
| 181 |
+
t = float(ev["time"])
|
| 182 |
+
dur = float(ev["duration"])
|
| 183 |
+
vol = float(ev.get("volume", 1.0))
|
| 184 |
+
|
| 185 |
+
# Use glide only if previous note ended very recently
|
| 186 |
+
glide_freq = prev_freq if (t - prev_end) < 0.25 else None
|
| 187 |
+
|
| 188 |
+
note_audio = self.note(
|
| 189 |
+
freq, dur,
|
| 190 |
+
prev_freq=glide_freq,
|
| 191 |
+
volume=vol,
|
| 192 |
+
)
|
| 193 |
+
track = add_at(track, note_audio, t)
|
| 194 |
+
prev_freq = freq
|
| 195 |
+
prev_end = t + dur
|
| 196 |
+
|
| 197 |
+
return track
|
synths/piano.py
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Piano synthesizer β soft felt-piano lullaby tone.
|
| 3 |
+
|
| 4 |
+
Uses additive synthesis with a small handful of inharmonic partials, each
|
| 5 |
+
with its own decay envelope (higher partials decay faster β that's what
|
| 6 |
+
makes a piano sound like a piano vs an organ). A short stiff-attack
|
| 7 |
+
transient gives the hammer strike; a long sustain gives the body.
|
| 8 |
+
|
| 9 |
+
The "felt-piano" sound (a piano with felt strips between hammer and string)
|
| 10 |
+
is the soft, intimate, lullaby-friendly variant β less attack, more body,
|
| 11 |
+
fewer harmonics.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
from scipy.signal import lfilter, butter
|
| 16 |
+
|
| 17 |
+
SR = 44100
|
| 18 |
+
|
| 19 |
+
NOTE = {
|
| 20 |
+
"C": 0, "C#": 1, "Db": 1, "D": 2, "D#": 3, "Eb": 3, "E": 4, "F": 5,
|
| 21 |
+
"F#": 6, "Gb": 6, "G": 7, "G#": 8, "Ab": 8, "A": 9, "A#": 10, "Bb": 10, "B": 11,
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
# Chord voicings: (note offsets in semitones from root) at the given octave
|
| 25 |
+
CHORD_INTERVALS = {
|
| 26 |
+
"": [0, 4, 7], # major
|
| 27 |
+
"m": [0, 3, 7], # minor
|
| 28 |
+
"7": [0, 4, 7, 10], # dom 7
|
| 29 |
+
"m7": [0, 3, 7, 10],
|
| 30 |
+
"maj7": [0, 4, 7, 11],
|
| 31 |
+
"sus2": [0, 2, 7],
|
| 32 |
+
"sus4": [0, 5, 7],
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def parse_chord(name):
|
| 37 |
+
"""'Am' β ('A', 'm'). 'C' β ('C', ''). 'Bbm7' β ('Bb', 'm7')."""
|
| 38 |
+
name = name.strip()
|
| 39 |
+
# Match 1-2 char root, rest is quality
|
| 40 |
+
if len(name) >= 2 and name[1] in ("#", "b"):
|
| 41 |
+
root, quality = name[:2], name[2:]
|
| 42 |
+
else:
|
| 43 |
+
root, quality = name[:1], name[1:]
|
| 44 |
+
return root, quality
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def note_to_hz(name):
|
| 48 |
+
"""e.g. 'A4' β 440.0"""
|
| 49 |
+
name = name.strip()
|
| 50 |
+
for i, ch in enumerate(name):
|
| 51 |
+
if ch.isdigit() or ch == "-":
|
| 52 |
+
pitch = name[:i]
|
| 53 |
+
octave = int(name[i:])
|
| 54 |
+
break
|
| 55 |
+
else:
|
| 56 |
+
pitch, octave = name, 4
|
| 57 |
+
midi = 12 * (octave + 1) + NOTE[pitch]
|
| 58 |
+
return 440.0 * 2 ** ((midi - 69) / 12)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def chord_to_freqs(chord_name, octave=4):
|
| 62 |
+
"""Return list of (frequency, relative_volume) for each note in the chord."""
|
| 63 |
+
root, quality = parse_chord(chord_name)
|
| 64 |
+
intervals = CHORD_INTERVALS.get(quality, CHORD_INTERVALS[""])
|
| 65 |
+
root_midi = 12 * (octave + 1) + NOTE[root]
|
| 66 |
+
|
| 67 |
+
notes = []
|
| 68 |
+
for i, interval in enumerate(intervals):
|
| 69 |
+
midi = root_midi + interval
|
| 70 |
+
hz = 440.0 * 2 ** ((midi - 69) / 12)
|
| 71 |
+
# Soften higher voices a touch β top of the chord shouldn't dominate
|
| 72 |
+
vol = 1.0 if i == 0 else (0.7 if i < 3 else 0.55)
|
| 73 |
+
notes.append((hz, vol))
|
| 74 |
+
return notes
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def add_at(track, audio, start_sec):
|
| 78 |
+
start = int(start_sec * SR)
|
| 79 |
+
end = start + len(audio)
|
| 80 |
+
if end > len(track):
|
| 81 |
+
track = np.pad(track, (0, end - len(track)))
|
| 82 |
+
track[start:end] += audio
|
| 83 |
+
return track
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
class PianoSynth:
|
| 87 |
+
"""Soft felt-piano synthesizer using additive synthesis."""
|
| 88 |
+
|
| 89 |
+
def __init__(self, sr=SR):
|
| 90 |
+
self.sr = sr
|
| 91 |
+
|
| 92 |
+
def note(self, freq, duration_s, volume=1.0, brightness=0.5):
|
| 93 |
+
"""
|
| 94 |
+
One piano note via additive synthesis.
|
| 95 |
+
|
| 96 |
+
brightness: 0..1 β controls how much energy is in the upper partials.
|
| 97 |
+
0.3 = warm/felt, 0.7 = bright/grand. Lullabies want ~0.4.
|
| 98 |
+
"""
|
| 99 |
+
sr = self.sr
|
| 100 |
+
n = int(duration_s * sr)
|
| 101 |
+
if n <= 0:
|
| 102 |
+
return np.zeros(0, dtype=np.float32)
|
| 103 |
+
t = np.arange(n) / sr
|
| 104 |
+
|
| 105 |
+
# Partial amplitudes β first 6 harmonics with falloff
|
| 106 |
+
# Brightness shifts the energy curve
|
| 107 |
+
base_amps = np.array([1.0, 0.55, 0.35, 0.22, 0.14, 0.08], dtype=np.float32)
|
| 108 |
+
# Apply brightness as a tilt β more brightness = less rolloff on highs
|
| 109 |
+
amps = base_amps * (1.0 + brightness * np.arange(len(base_amps)) * 0.1)
|
| 110 |
+
amps[0] = 1.0 # keep fundamental at unity
|
| 111 |
+
|
| 112 |
+
# Piano partials are slightly inharmonic (stiffness of real strings).
|
| 113 |
+
# The inharmonicity coefficient grows roughly as nΒ² but is small for
|
| 114 |
+
# the low-mid range β ~0.0004 for a real piano.
|
| 115 |
+
B = 0.0004
|
| 116 |
+
|
| 117 |
+
signal = np.zeros(n, dtype=np.float32)
|
| 118 |
+
for k in range(len(amps)):
|
| 119 |
+
partial = k + 1
|
| 120 |
+
# Inharmonic frequency
|
| 121 |
+
partial_freq = freq * partial * np.sqrt(1 + B * partial ** 2)
|
| 122 |
+
if partial_freq > sr / 2 * 0.95:
|
| 123 |
+
break
|
| 124 |
+
|
| 125 |
+
# Each partial has its own decay β higher partials die faster.
|
| 126 |
+
# Felt piano: fundamental rings ~4-6s, partial 6 dies in <1s.
|
| 127 |
+
decay_time = duration_s * (1.0 - 0.12 * k)
|
| 128 |
+
decay = np.exp(-t / max(decay_time, 0.2))
|
| 129 |
+
|
| 130 |
+
# Slight pitch detune per partial (real pianos have multiple strings
|
| 131 |
+
# per note with tiny detune that creates the chorus shimmer)
|
| 132 |
+
detune = 1.0 + (np.random.randn() * 0.0005 if k == 0 else 0)
|
| 133 |
+
phase = 2 * np.pi * partial_freq * detune * t
|
| 134 |
+
|
| 135 |
+
signal += amps[k] * decay * np.sin(phase)
|
| 136 |
+
|
| 137 |
+
# Hammer strike: very short noise burst at the start, lowpassed.
|
| 138 |
+
# This is what makes it sound percussive instead of bowed.
|
| 139 |
+
attack_n = min(int(0.012 * sr), n) # 12ms
|
| 140 |
+
strike = np.random.randn(attack_n).astype(np.float32) * 0.15
|
| 141 |
+
# Lowpass the strike so it's a thud, not a click
|
| 142 |
+
b_lp, a_lp = butter(2, 2500 / (sr / 2), btype="low")
|
| 143 |
+
strike = lfilter(b_lp, a_lp, strike).astype(np.float32)
|
| 144 |
+
strike_env = np.exp(-np.arange(attack_n) / (sr * 0.005))
|
| 145 |
+
signal[:attack_n] += (strike * strike_env).astype(np.float32)
|
| 146 |
+
|
| 147 |
+
# Soft attack envelope on the body (no clicks)
|
| 148 |
+
atk_n = min(int(0.008 * sr), n)
|
| 149 |
+
if atk_n > 1:
|
| 150 |
+
signal[:atk_n] *= np.linspace(0, 1, atk_n)
|
| 151 |
+
|
| 152 |
+
# Soft release to avoid clipping silence at end
|
| 153 |
+
rel_n = min(int(0.08 * sr), n // 2)
|
| 154 |
+
if rel_n > 1:
|
| 155 |
+
signal[-rel_n:] *= np.linspace(1, 0, rel_n)
|
| 156 |
+
|
| 157 |
+
# Gentle lowpass for "felt" character
|
| 158 |
+
b_warm, a_warm = butter(1, 6000 / (sr / 2), btype="low")
|
| 159 |
+
signal = lfilter(b_warm, a_warm, signal).astype(np.float32)
|
| 160 |
+
|
| 161 |
+
# Normalize per-note then scale by volume
|
| 162 |
+
peak = float(np.max(np.abs(signal)))
|
| 163 |
+
if peak > 1e-9:
|
| 164 |
+
signal = signal / peak * 0.85
|
| 165 |
+
return signal * volume
|
| 166 |
+
|
| 167 |
+
def chord(self, chord_name, duration_s, time, octave=4,
|
| 168 |
+
direction="down", spread_ms=20, volume=0.7, brightness=0.4):
|
| 169 |
+
"""
|
| 170 |
+
Render a chord as multiple piano notes with a tiny stagger.
|
| 171 |
+
|
| 172 |
+
For lullaby use, spread_ms=20-40 gives a gentle "rolled" feel like
|
| 173 |
+
a pianist landing slightly arpeggiated.
|
| 174 |
+
"""
|
| 175 |
+
notes = chord_to_freqs(chord_name, octave=octave)
|
| 176 |
+
if direction == "up":
|
| 177 |
+
notes = list(reversed(notes))
|
| 178 |
+
|
| 179 |
+
# Time-stagger via the spread
|
| 180 |
+
stagger = spread_ms / 1000.0
|
| 181 |
+
chord_audio = np.zeros(int((duration_s + stagger * len(notes) + 0.5) * self.sr),
|
| 182 |
+
dtype=np.float32)
|
| 183 |
+
|
| 184 |
+
for i, (freq, rel_vol) in enumerate(notes):
|
| 185 |
+
note_audio = self.note(freq, duration_s, volume=volume * rel_vol,
|
| 186 |
+
brightness=brightness)
|
| 187 |
+
start = int(i * stagger * self.sr)
|
| 188 |
+
end = start + len(note_audio)
|
| 189 |
+
if end > len(chord_audio):
|
| 190 |
+
chord_audio = np.pad(chord_audio, (0, end - len(chord_audio)))
|
| 191 |
+
chord_audio[start:end] += note_audio
|
| 192 |
+
return chord_audio
|
| 193 |
+
|
| 194 |
+
def sequence(self, events):
|
| 195 |
+
"""
|
| 196 |
+
Render a sequence of chord and/or note events.
|
| 197 |
+
|
| 198 |
+
chord event: {"type": "chord", "name": "Am", "time": 0.0, "duration": 3.0, ...}
|
| 199 |
+
note event: {"type": "note", "name": "A4", "time": 0.0, "duration": 1.0, ...}
|
| 200 |
+
"""
|
| 201 |
+
if not events:
|
| 202 |
+
return np.zeros(int(self.sr), dtype=np.float32)
|
| 203 |
+
|
| 204 |
+
events = sorted(events, key=lambda e: e["time"])
|
| 205 |
+
end_time = max(e["time"] + e["duration"] for e in events) + 2.0
|
| 206 |
+
track = np.zeros(int(end_time * self.sr) + 1, dtype=np.float32)
|
| 207 |
+
|
| 208 |
+
for ev in events:
|
| 209 |
+
typ = ev.get("type")
|
| 210 |
+
if typ == "chord":
|
| 211 |
+
audio = self.chord(
|
| 212 |
+
ev["name"],
|
| 213 |
+
ev["duration"],
|
| 214 |
+
ev["time"],
|
| 215 |
+
octave=ev.get("octave", 4),
|
| 216 |
+
direction=ev.get("direction", "down"),
|
| 217 |
+
spread_ms=ev.get("spread_ms", 25),
|
| 218 |
+
volume=ev.get("volume", 0.7),
|
| 219 |
+
brightness=ev.get("brightness", 0.4),
|
| 220 |
+
)
|
| 221 |
+
elif typ == "note":
|
| 222 |
+
freq = note_to_hz(ev["name"])
|
| 223 |
+
# Piano notes ring out β extend duration for natural decay tail
|
| 224 |
+
audio = self.note(
|
| 225 |
+
freq,
|
| 226 |
+
ev["duration"] + 1.0,
|
| 227 |
+
volume=ev.get("volume", 0.85),
|
| 228 |
+
brightness=ev.get("brightness", 0.4),
|
| 229 |
+
)
|
| 230 |
+
else:
|
| 231 |
+
continue
|
| 232 |
+
track = add_at(track, audio, ev["time"])
|
| 233 |
+
|
| 234 |
+
return track
|
synths/voice.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Voice synthesis for the lullaby app β Kokoro TTS, soft preset.
|
| 3 |
+
|
| 4 |
+
Reads the lyrics gently, slowly, with natural breaths between lines.
|
| 5 |
+
No singing, no pitch manipulation β just a warm reading voice over music.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
|
| 10 |
+
SR_TARGET = 44100
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
# We always use the soft voice β chosen for lullaby use.
|
| 14 |
+
VOICE_DEFAULT = "af_nicole"
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
_pipeline = None
|
| 18 |
+
_kokoro_sr = 24000
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def _load_kokoro():
|
| 22 |
+
global _pipeline
|
| 23 |
+
if _pipeline is not None:
|
| 24 |
+
return True
|
| 25 |
+
try:
|
| 26 |
+
from kokoro import KPipeline
|
| 27 |
+
print("Loading Kokoro (first run downloads ~80MB)...")
|
| 28 |
+
_pipeline = KPipeline(lang_code='a', repo_id='hexgrad/Kokoro-82M')
|
| 29 |
+
print("Kokoro loaded.")
|
| 30 |
+
return True
|
| 31 |
+
except Exception as e:
|
| 32 |
+
print(f"Kokoro unavailable: {e}")
|
| 33 |
+
return False
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _resample(x, sr_from, sr_to):
|
| 37 |
+
if sr_from == sr_to:
|
| 38 |
+
return x
|
| 39 |
+
n_out = int(len(x) * sr_to / sr_from)
|
| 40 |
+
return np.interp(
|
| 41 |
+
np.linspace(0, len(x) - 1, n_out),
|
| 42 |
+
np.arange(len(x)),
|
| 43 |
+
x,
|
| 44 |
+
).astype(np.float32)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _silence(seconds):
|
| 48 |
+
return np.zeros(int(seconds * SR_TARGET), dtype=np.float32)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _gentle_vocal_eq(audio):
|
| 52 |
+
"""Soften vocal tone β gentle low-pass + mild high-pass."""
|
| 53 |
+
from scipy.signal import butter, lfilter
|
| 54 |
+
b_lp, a_lp = butter(2, 5500 / (SR_TARGET / 2), btype="low")
|
| 55 |
+
b_hp, a_hp = butter(1, 110 / (SR_TARGET / 2), btype="high")
|
| 56 |
+
out = lfilter(b_lp, a_lp, audio)
|
| 57 |
+
out = lfilter(b_hp, a_hp, out)
|
| 58 |
+
return out.astype(np.float32)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def speak_lyrics(lyrics, target_seconds=None, speed=0.85):
|
| 62 |
+
"""
|
| 63 |
+
Render lyrics as gentle spoken voice. Returns mono float32 at SR_TARGET.
|
| 64 |
+
|
| 65 |
+
speed=0.85 β slightly slower than normal, bedtime pacing.
|
| 66 |
+
target_seconds (if given) β pad with intro silence so voice ends near track end.
|
| 67 |
+
"""
|
| 68 |
+
if not _load_kokoro():
|
| 69 |
+
print("WARNING: TTS unavailable, returning silent vocal track")
|
| 70 |
+
return _silence(target_seconds or 1.0)
|
| 71 |
+
|
| 72 |
+
try:
|
| 73 |
+
chunks = []
|
| 74 |
+
generator = _pipeline(
|
| 75 |
+
lyrics,
|
| 76 |
+
voice=VOICE_DEFAULT,
|
| 77 |
+
speed=speed,
|
| 78 |
+
split_pattern=r'\n+',
|
| 79 |
+
)
|
| 80 |
+
for _, _, audio in generator:
|
| 81 |
+
audio_np = np.asarray(audio, dtype=np.float32)
|
| 82 |
+
if len(audio_np) == 0:
|
| 83 |
+
continue
|
| 84 |
+
chunks.append(audio_np)
|
| 85 |
+
# 350ms breath between phrases
|
| 86 |
+
chunks.append(np.zeros(int(0.35 * _kokoro_sr), dtype=np.float32))
|
| 87 |
+
if not chunks:
|
| 88 |
+
return _silence(target_seconds or 1.0)
|
| 89 |
+
audio = np.concatenate(chunks)
|
| 90 |
+
audio = _resample(audio, _kokoro_sr, SR_TARGET)
|
| 91 |
+
audio = _gentle_vocal_eq(audio)
|
| 92 |
+
|
| 93 |
+
# Pad with intro silence if requested
|
| 94 |
+
if target_seconds is not None:
|
| 95 |
+
target_n = int(target_seconds * SR_TARGET)
|
| 96 |
+
if len(audio) < target_n:
|
| 97 |
+
intro = min(target_n - len(audio), 4 * SR_TARGET)
|
| 98 |
+
audio = np.concatenate([_silence(intro / SR_TARGET), audio])
|
| 99 |
+
return audio
|
| 100 |
+
except Exception as e:
|
| 101 |
+
print(f"Kokoro generation error: {e}")
|
| 102 |
+
return _silence(target_seconds or 1.0)
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
# Back-compat alias for code that imports the old name.
|
| 106 |
+
def synthesize_singing(lyrics, target_seconds=20.0, voice_style="soft", **kwargs):
|
| 107 |
+
return speak_lyrics(lyrics, target_seconds=target_seconds)
|
synths/xylophone.py
ADDED
|
@@ -0,0 +1,298 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Xylophone synthesizer β based on spectral analysis of a real soft-mallet
|
| 3 |
+
metallophone/xylophone loop reference.
|
| 4 |
+
|
| 5 |
+
Analysis of the reference (van_wiese xylophone loop, 125 BPM):
|
| 6 |
+
- Fundamental at 463 Hz (A#4) with 2nd harmonic at 931 Hz
|
| 7 |
+
- Amplitude ratio fundamental:2nd β 32:1 (essentially a pure sine + whisper of H2)
|
| 8 |
+
- Decay to -6dB in 86ms
|
| 9 |
+
- Decay to -20dB in ~300ms
|
| 10 |
+
- No prominent inharmonic partials (cleaner than a true wooden xylophone)
|
| 11 |
+
- Bright, near-pure tone β closer to a tuned soft-mallet metallophone
|
| 12 |
+
|
| 13 |
+
This synth models that: an almost-sine fundamental, a faint 2nd harmonic,
|
| 14 |
+
a tiny inharmonic shimmer for "real wood/metal" character, a brief filtered
|
| 15 |
+
strike transient, and a fast exponential decay.
|
| 16 |
+
|
| 17 |
+
Discrete hits, no tremolo, no roll β single mallet strikes like the reference.
|
| 18 |
+
For chord roles, we play a fast arpeggio of single hits instead of a tremolo.
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
import numpy as np
|
| 22 |
+
from scipy.signal import lfilter, butter
|
| 23 |
+
|
| 24 |
+
SR = 44100
|
| 25 |
+
|
| 26 |
+
NOTE = {
|
| 27 |
+
"C": 0, "C#": 1, "Db": 1, "D": 2, "D#": 3, "Eb": 3, "E": 4, "F": 5,
|
| 28 |
+
"F#": 6, "Gb": 6, "G": 7, "G#": 8, "Ab": 8, "A": 9, "A#": 10, "Bb": 10, "B": 11,
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
CHORD_INTERVALS = {
|
| 32 |
+
"": [0, 4, 7],
|
| 33 |
+
"m": [0, 3, 7],
|
| 34 |
+
"7": [0, 4, 7, 10],
|
| 35 |
+
"m7": [0, 3, 7, 10],
|
| 36 |
+
"maj7": [0, 4, 7, 11],
|
| 37 |
+
"sus2": [0, 2, 7],
|
| 38 |
+
"sus4": [0, 5, 7],
|
| 39 |
+
}
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def parse_chord(name):
|
| 43 |
+
name = name.strip()
|
| 44 |
+
if len(name) >= 2 and name[1] in ("#", "b"):
|
| 45 |
+
root, quality = name[:2], name[2:]
|
| 46 |
+
else:
|
| 47 |
+
root, quality = name[:1], name[1:]
|
| 48 |
+
return root, quality
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def note_to_hz(name):
|
| 52 |
+
"""'A5' β 880.0"""
|
| 53 |
+
name = name.strip()
|
| 54 |
+
for i, ch in enumerate(name):
|
| 55 |
+
if ch.isdigit() or ch == "-":
|
| 56 |
+
pitch = name[:i]
|
| 57 |
+
octave = int(name[i:])
|
| 58 |
+
break
|
| 59 |
+
else:
|
| 60 |
+
pitch, octave = name, 5
|
| 61 |
+
midi = 12 * (octave + 1) + NOTE[pitch]
|
| 62 |
+
return 440.0 * 2 ** ((midi - 69) / 12)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def chord_to_freqs(chord_name, octave=4):
|
| 66 |
+
root, quality = parse_chord(chord_name)
|
| 67 |
+
intervals = CHORD_INTERVALS.get(quality, CHORD_INTERVALS[""])
|
| 68 |
+
root_midi = 12 * (octave + 1) + NOTE[root]
|
| 69 |
+
return [440.0 * 2 ** ((root_midi + i - 69) / 12) for i in intervals]
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def add_at(track, audio, start_sec):
|
| 73 |
+
start = int(start_sec * SR)
|
| 74 |
+
end = start + len(audio)
|
| 75 |
+
if end > len(track):
|
| 76 |
+
track = np.pad(track, (0, end - len(track)))
|
| 77 |
+
track[start:end] += audio
|
| 78 |
+
return track
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def simple_reverb(audio, sr, room_size=0.5, damping=0.55, wet=0.30):
|
| 82 |
+
"""
|
| 83 |
+
Multi-tap delay reverb. Adds ~1.5s tail with diffusion.
|
| 84 |
+
|
| 85 |
+
For xylophone we want more wet than the harp (xylo strikes are short
|
| 86 |
+
and dry sounds toy-like). 6 lowpassed delayed taps + a recirculating
|
| 87 |
+
feedback delay build a smooth tail without metallic resonance.
|
| 88 |
+
|
| 89 |
+
Parameters:
|
| 90 |
+
room_size: 0..1 β scales tap delays (bigger = longer tail)
|
| 91 |
+
damping: 0..1 β lowpass cutoff scaling (more = darker tail)
|
| 92 |
+
wet: 0..1 β wet/dry mix
|
| 93 |
+
"""
|
| 94 |
+
if wet <= 0.0:
|
| 95 |
+
return audio
|
| 96 |
+
|
| 97 |
+
n = len(audio)
|
| 98 |
+
base_delays_ms = [29.0, 37.0, 53.0, 67.0, 89.0, 113.0]
|
| 99 |
+
gains = [0.55, 0.48, 0.42, 0.36, 0.30, 0.24]
|
| 100 |
+
|
| 101 |
+
wet_buf = np.zeros(n + int(sr * 2.0), dtype=np.float32)
|
| 102 |
+
|
| 103 |
+
cutoff = max(800.0, 5000.0 * (1.0 - damping))
|
| 104 |
+
b_lp, a_lp = butter(2, cutoff / (sr / 2), btype="low")
|
| 105 |
+
pre_lp = lfilter(b_lp, a_lp, audio).astype(np.float32)
|
| 106 |
+
|
| 107 |
+
for ms, g in zip(base_delays_ms, gains):
|
| 108 |
+
delay_samples = int(ms * sr / 1000.0 * (0.6 + room_size * 0.8))
|
| 109 |
+
end = delay_samples + n
|
| 110 |
+
if end > len(wet_buf):
|
| 111 |
+
wet_buf = np.pad(wet_buf, (0, end - len(wet_buf)))
|
| 112 |
+
wet_buf[delay_samples:end] += pre_lp * g
|
| 113 |
+
|
| 114 |
+
fb_delay = int(0.071 * sr * (0.6 + room_size * 0.8))
|
| 115 |
+
fb_gain = 0.45 + room_size * 0.20
|
| 116 |
+
for k in range(1, 5):
|
| 117 |
+
offset = fb_delay * k
|
| 118 |
+
if offset >= len(wet_buf):
|
| 119 |
+
break
|
| 120 |
+
wet_buf[offset:] += wet_buf[:-offset] * (fb_gain ** k) * 0.5
|
| 121 |
+
|
| 122 |
+
wet_buf = wet_buf[:n]
|
| 123 |
+
wet_buf = lfilter(b_lp, a_lp, wet_buf).astype(np.float32)
|
| 124 |
+
|
| 125 |
+
return (audio * (1.0 - wet) + wet_buf * wet).astype(np.float32)
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
class XylophoneSynth:
|
| 129 |
+
"""
|
| 130 |
+
Soft-mallet metallophone/xylophone modeled on a real reference loop.
|
| 131 |
+
|
| 132 |
+
Characteristics:
|
| 133 |
+
- Near-pure sine fundamental
|
| 134 |
+
- Whisper of 2nd harmonic (~1/30 the amplitude of fundamental)
|
| 135 |
+
- Tiny inharmonic flavor for body
|
| 136 |
+
- Fast exponential decay (~85ms to -6dB, ~300ms to -20dB)
|
| 137 |
+
- Brief click-like strike transient
|
| 138 |
+
"""
|
| 139 |
+
|
| 140 |
+
def __init__(self, sr=SR):
|
| 141 |
+
self.sr = sr
|
| 142 |
+
|
| 143 |
+
def strike(self, freq, volume=1.0, brightness=0.5):
|
| 144 |
+
"""
|
| 145 |
+
One mallet strike.
|
| 146 |
+
|
| 147 |
+
Models the reference envelope: brief attack peak, ~100ms near-peak
|
| 148 |
+
plateau (bar body sustain), then slow exponential decay over ~600ms.
|
| 149 |
+
Full ring-out by ~800ms.
|
| 150 |
+
|
| 151 |
+
brightness: 0..1 β affects strike click intensity and H2 level.
|
| 152 |
+
0.5 matches the reference well.
|
| 153 |
+
"""
|
| 154 |
+
sr = self.sr
|
| 155 |
+
# 900ms render β long enough for the full decay tail
|
| 156 |
+
n = int(0.9 * sr)
|
| 157 |
+
t = np.arange(n) / sr
|
| 158 |
+
|
| 159 |
+
# Reference fundamental envelope: stays near-peak until ~140ms, then
|
| 160 |
+
# decays to ~0.25 by 300ms, ~0.10 by 500ms. That's a slow exponential
|
| 161 |
+
# with tau β 220ms, but with a brief plateau at the start.
|
| 162 |
+
#
|
| 163 |
+
# Model: attack ramp β short plateau β exponential decay
|
| 164 |
+
plateau_end = 0.12 # 120ms plateau
|
| 165 |
+
decay_tau = 0.22 # slow decay constant
|
| 166 |
+
|
| 167 |
+
fund_env = np.ones(n, dtype=np.float32)
|
| 168 |
+
# Plateau phase: gentle taper from 1.0 to 0.95
|
| 169 |
+
plateau_n = int(plateau_end * sr)
|
| 170 |
+
fund_env[:plateau_n] = np.linspace(1.0, 0.95, plateau_n)
|
| 171 |
+
# Decay phase: exponential from 0.95 down
|
| 172 |
+
decay_n = n - plateau_n
|
| 173 |
+
decay_t = np.arange(decay_n) / sr
|
| 174 |
+
fund_env[plateau_n:] = 0.95 * np.exp(-decay_t / decay_tau)
|
| 175 |
+
|
| 176 |
+
# H2 β barely audible, dies faster than fundamental (no plateau)
|
| 177 |
+
h2_amp = 0.04 + brightness * 0.025
|
| 178 |
+
tau_h2 = 0.090
|
| 179 |
+
h2_env = np.exp(-t / tau_h2)
|
| 180 |
+
|
| 181 |
+
signal = (np.sin(2 * np.pi * freq * t) * fund_env
|
| 182 |
+
+ h2_amp * np.sin(2 * np.pi * freq * 2 * t) * h2_env)
|
| 183 |
+
|
| 184 |
+
# Faint inharmonic shimmer at attack only β gives "real bar" texture
|
| 185 |
+
partial_3_freq = freq * 4.2
|
| 186 |
+
if partial_3_freq < sr / 2 * 0.9:
|
| 187 |
+
partial_3_env = np.exp(-t / 0.025)
|
| 188 |
+
partial_3_amp = 0.04 * brightness
|
| 189 |
+
signal += partial_3_amp * np.sin(2 * np.pi * partial_3_freq * t) * partial_3_env
|
| 190 |
+
|
| 191 |
+
# Strike transient: brief filtered noise burst.
|
| 192 |
+
attack_n = int(0.006 * sr) # 6ms
|
| 193 |
+
if attack_n > 0:
|
| 194 |
+
noise = np.random.randn(attack_n).astype(np.float32) * 0.10
|
| 195 |
+
b, a = butter(2,
|
| 196 |
+
[1500 / (sr / 2), min(0.95, 4500 / (sr / 2))],
|
| 197 |
+
btype="band")
|
| 198 |
+
click = lfilter(b, a, noise).astype(np.float32)
|
| 199 |
+
click_env = np.exp(-np.arange(attack_n) / (sr * 0.002))
|
| 200 |
+
click = (click * click_env * (0.5 + brightness * 0.4)).astype(np.float32)
|
| 201 |
+
signal[:attack_n] += click
|
| 202 |
+
|
| 203 |
+
# Tiny attack ramp on the body (no DC pop)
|
| 204 |
+
ramp_n = int(0.001 * sr)
|
| 205 |
+
if ramp_n > 1:
|
| 206 |
+
signal[:ramp_n] *= np.linspace(0, 1, ramp_n)
|
| 207 |
+
|
| 208 |
+
# Normalize per-strike to a known peak, then scale by volume
|
| 209 |
+
peak = float(np.max(np.abs(signal)))
|
| 210 |
+
if peak > 1e-9:
|
| 211 |
+
signal = signal / peak * 0.80
|
| 212 |
+
|
| 213 |
+
return (signal * volume).astype(np.float32)
|
| 214 |
+
|
| 215 |
+
def chord(self, chord_name, duration_s, octave=4,
|
| 216 |
+
volume=0.7, brightness=0.5,
|
| 217 |
+
arpeggio_ms=60, direction="up"):
|
| 218 |
+
"""
|
| 219 |
+
Render a chord as a quick arpeggio of individual strikes (not a roll).
|
| 220 |
+
|
| 221 |
+
Real mallet players play chords as fast arpeggios from low to high
|
| 222 |
+
(or high to low). arpeggio_ms is the delay between strikes. 40-80ms
|
| 223 |
+
feels right for lullaby pacing.
|
| 224 |
+
|
| 225 |
+
The 'duration_s' is the chord's total time-slot, but each strike
|
| 226 |
+
decays on its own ~500ms timeline regardless.
|
| 227 |
+
"""
|
| 228 |
+
freqs = chord_to_freqs(chord_name, octave=octave)
|
| 229 |
+
if direction == "down":
|
| 230 |
+
freqs = list(reversed(freqs))
|
| 231 |
+
|
| 232 |
+
stagger = arpeggio_ms / 1000.0
|
| 233 |
+
total_len = int((duration_s + 0.6) * self.sr)
|
| 234 |
+
out = np.zeros(total_len, dtype=np.float32)
|
| 235 |
+
|
| 236 |
+
for i, freq in enumerate(freqs):
|
| 237 |
+
# Voicing: root slightly louder, top voice slightly quieter
|
| 238 |
+
voice_vol = 1.0 if i == 0 else (0.78 if i < 3 else 0.6)
|
| 239 |
+
strike = self.strike(freq,
|
| 240 |
+
volume=volume * voice_vol,
|
| 241 |
+
brightness=brightness)
|
| 242 |
+
start = int(i * stagger * self.sr)
|
| 243 |
+
end = start + len(strike)
|
| 244 |
+
if end > len(out):
|
| 245 |
+
out = np.pad(out, (0, end - len(out)))
|
| 246 |
+
out[start:end] += strike
|
| 247 |
+
return out
|
| 248 |
+
|
| 249 |
+
def note(self, freq, duration_s, volume=1.0, brightness=0.5):
|
| 250 |
+
"""
|
| 251 |
+
Single mallet strike for melody use.
|
| 252 |
+
|
| 253 |
+
duration_s is ignored beyond a minimum β the strike has its own
|
| 254 |
+
natural decay envelope (~400ms). Don't try to sustain it.
|
| 255 |
+
"""
|
| 256 |
+
return self.strike(freq, volume=volume, brightness=brightness)
|
| 257 |
+
|
| 258 |
+
def sequence(self, events):
|
| 259 |
+
"""
|
| 260 |
+
Render mixed events: 'chord' for rhythm role (arpeggiated),
|
| 261 |
+
'note' for melody role (single strikes).
|
| 262 |
+
"""
|
| 263 |
+
if not events:
|
| 264 |
+
return np.zeros(int(self.sr), dtype=np.float32)
|
| 265 |
+
|
| 266 |
+
events = sorted(events, key=lambda e: e["time"])
|
| 267 |
+
# Extend tail to capture the reverb decay (~2s after the last event)
|
| 268 |
+
end_time = max(e["time"] + max(e["duration"], 0.6) for e in events) + 2.0
|
| 269 |
+
track = np.zeros(int(end_time * self.sr) + 1, dtype=np.float32)
|
| 270 |
+
|
| 271 |
+
for ev in events:
|
| 272 |
+
typ = ev.get("type")
|
| 273 |
+
if typ == "chord":
|
| 274 |
+
audio = self.chord(
|
| 275 |
+
ev["name"], ev["duration"],
|
| 276 |
+
octave=ev.get("octave", 4),
|
| 277 |
+
volume=ev.get("volume", 0.7),
|
| 278 |
+
brightness=ev.get("brightness", 0.5),
|
| 279 |
+
arpeggio_ms=ev.get("arpeggio_ms", 60),
|
| 280 |
+
direction=ev.get("direction", "up"),
|
| 281 |
+
)
|
| 282 |
+
elif typ == "note":
|
| 283 |
+
freq = note_to_hz(ev["name"])
|
| 284 |
+
audio = self.note(
|
| 285 |
+
freq, ev["duration"],
|
| 286 |
+
volume=ev.get("volume", 1.0),
|
| 287 |
+
brightness=ev.get("brightness", 0.5),
|
| 288 |
+
)
|
| 289 |
+
else:
|
| 290 |
+
continue
|
| 291 |
+
track = add_at(track, audio, ev["time"])
|
| 292 |
+
|
| 293 |
+
# Apply reverb at the mix level β xylo strikes are short and dry,
|
| 294 |
+
# so a more generous wet ratio than the harp makes them sit
|
| 295 |
+
# naturally in a "room" rather than feeling like a toy.
|
| 296 |
+
track = simple_reverb(track, self.sr,
|
| 297 |
+
room_size=0.45, damping=0.55, wet=0.32)
|
| 298 |
+
return track
|
tests/test_audio_pipeline.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Test the full audio pipeline end-to-end with a hardcoded lullaby.
|
| 3 |
+
No LLM needed β uses a fixed parsed lullaby structure.
|
| 4 |
+
|
| 5 |
+
Tests every (rhythm Γ melody) instrument pair. Use --no-voice to skip Kokoro.
|
| 6 |
+
|
| 7 |
+
Run:
|
| 8 |
+
python test_audio_pipeline.py
|
| 9 |
+
python test_audio_pipeline.py --no-voice
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
import sys
|
| 13 |
+
import time
|
| 14 |
+
import numpy as np
|
| 15 |
+
import soundfile as sf
|
| 16 |
+
|
| 17 |
+
from app import (
|
| 18 |
+
render_rhythm,
|
| 19 |
+
render_melody,
|
| 20 |
+
mix_tracks,
|
| 21 |
+
SR,
|
| 22 |
+
)
|
| 23 |
+
from voice import speak_lyrics
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
PARSED = {
|
| 27 |
+
"bpm": 60,
|
| 28 |
+
"progression": ["D", "Bm", "G", "A"],
|
| 29 |
+
"lines": [
|
| 30 |
+
[("D", "Lucy, dear Lucy,")],
|
| 31 |
+
[("Bm", "the dark wraps the earth now,")],
|
| 32 |
+
[("G", "the cat curls up near you,")],
|
| 33 |
+
[("A", "the moon hums a soft tune...")],
|
| 34 |
+
[("D", "The stars know your name now,")],
|
| 35 |
+
[("Bm", "they call from above,")],
|
| 36 |
+
[("G", "your blanket is gentle,")],
|
| 37 |
+
[("A", "sewn warm from our love...")],
|
| 38 |
+
],
|
| 39 |
+
"plain_lyrics": (
|
| 40 |
+
"Lucy, dear Lucy,\n"
|
| 41 |
+
"the dark wraps the earth now,\n"
|
| 42 |
+
"the cat curls up near you,\n"
|
| 43 |
+
"the moon hums a soft tune...\n"
|
| 44 |
+
"The stars know your name now,\n"
|
| 45 |
+
"they call from above,\n"
|
| 46 |
+
"your blanket is gentle,\n"
|
| 47 |
+
"sewn warm from our love..."
|
| 48 |
+
),
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
INSTRUMENTS = ["guitar", "piano", "xylophone", "ocarina"]
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def test_pair(rhythm_instr, melody_instr, do_voice=True):
|
| 55 |
+
label = f"{rhythm_instr}+{melody_instr}"
|
| 56 |
+
print(f"\n=== {label} ===")
|
| 57 |
+
|
| 58 |
+
t0 = time.time()
|
| 59 |
+
rhythm, total_s = render_rhythm(PARSED, rhythm_instr)
|
| 60 |
+
print(f" rhythm ({rhythm_instr}): {len(rhythm)/SR:.1f}s ({time.time()-t0:.1f}s gen)")
|
| 61 |
+
|
| 62 |
+
t0 = time.time()
|
| 63 |
+
melody = render_melody(PARSED, melody_instr)
|
| 64 |
+
print(f" melody ({melody_instr}): {len(melody)/SR:.1f}s ({time.time()-t0:.1f}s gen)")
|
| 65 |
+
|
| 66 |
+
# Stem: rhythm + melody only
|
| 67 |
+
n = max(len(rhythm), len(melody))
|
| 68 |
+
rm = np.zeros(n, dtype=np.float32)
|
| 69 |
+
rm[:len(rhythm)] += rhythm * 0.7
|
| 70 |
+
rm[:len(melody)] += melody * 0.32
|
| 71 |
+
peak = np.abs(rm).max()
|
| 72 |
+
if peak > 0.95: rm *= 0.95 / peak
|
| 73 |
+
sf.write(f"music_test_output/test_{label}.wav", rm, SR, subtype="PCM_16")
|
| 74 |
+
print(f" β test_{label}.wav")
|
| 75 |
+
|
| 76 |
+
if not do_voice:
|
| 77 |
+
return
|
| 78 |
+
|
| 79 |
+
t0 = time.time()
|
| 80 |
+
voice = speak_lyrics(PARSED["plain_lyrics"], target_seconds=total_s)
|
| 81 |
+
print(f" voice: {len(voice)/SR:.1f}s ({time.time()-t0:.1f}s gen)")
|
| 82 |
+
|
| 83 |
+
mix = mix_tracks(rhythm, melody, voice)
|
| 84 |
+
sf.write(f"music_test_output/test_full_{label}.wav", mix, SR, subtype="PCM_16")
|
| 85 |
+
print(f" β test_full_{label}.wav")
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def main():
|
| 89 |
+
no_voice = "--no-voice" in sys.argv
|
| 90 |
+
|
| 91 |
+
# Test sensible default pairings (not all 16 combos β too many files)
|
| 92 |
+
pairs = [
|
| 93 |
+
("guitar", "ocarina"),
|
| 94 |
+
("guitar", "xylophone"),
|
| 95 |
+
("piano", "ocarina"),
|
| 96 |
+
("piano", "xylophone"),
|
| 97 |
+
("xylophone", "ocarina"),
|
| 98 |
+
("ocarina", "xylophone"),
|
| 99 |
+
]
|
| 100 |
+
for r, m in pairs:
|
| 101 |
+
test_pair(r, m, do_voice=not no_voice)
|
| 102 |
+
print("\ndone.")
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
if __name__ == "__main__":
|
| 106 |
+
main()
|
tests/test_llm.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Test the LLM + parser only. No audio, no Gradio, no Kokoro.
|
| 3 |
+
|
| 4 |
+
Usage:
|
| 5 |
+
python test_llm.py # uses LULLABY_GGUF env var or ./lullaby.gguf
|
| 6 |
+
LULLABY_GGUF=/path/to/x.gguf python test_llm.py
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
import os
|
| 10 |
+
import re
|
| 11 |
+
import sys
|
| 12 |
+
import time
|
| 13 |
+
|
| 14 |
+
from llama_cpp import Llama
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
MODEL_PATH = os.environ.get("LULLABY_GGUF", "lolaby.gguf")
|
| 18 |
+
|
| 19 |
+
SYSTEM_PROMPT = (
|
| 20 |
+
"You write personalized lullabies for small children, with chord markers "
|
| 21 |
+
"and a tempo/meter header so a guitar accompaniment can be rendered. "
|
| 22 |
+
"Output only the lullaby β no preamble."
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
# Same regexes as app.py β if these match, the audio pipeline will too.
|
| 26 |
+
TEMPO_RE = re.compile(r"Tempo:\s*(\d+)\s*bpm,\s*(\d+)/(\d+)", re.IGNORECASE)
|
| 27 |
+
PROG_RE = re.compile(r"Progression:\s*([A-G][^\n]+)", re.IGNORECASE)
|
| 28 |
+
CHORD_RE = re.compile(r"\[([A-G][^\]]*)\]\s*([^\[\n]*)")
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
TEST_PROMPTS = [
|
| 32 |
+
(
|
| 33 |
+
"Mia, sleepy and comforted, 6/8",
|
| 34 |
+
"Write a lullaby for: Mia, age 3\n"
|
| 35 |
+
"Loves: her stuffed elephant Pip, the moon\n"
|
| 36 |
+
"Mood: sleepy and comforted\n"
|
| 37 |
+
"Key: C major\n"
|
| 38 |
+
"Meter: 6/8",
|
| 39 |
+
),
|
| 40 |
+
(
|
| 41 |
+
"Theo with fear, minor key",
|
| 42 |
+
"Write a lullaby for: Theo, age 2\n"
|
| 43 |
+
"Loves: trains and tunnels\n"
|
| 44 |
+
"Fears: the dark\n"
|
| 45 |
+
"Mood: tearful, needs soothing\n"
|
| 46 |
+
"Key: A minor\n"
|
| 47 |
+
"Meter: 6/8",
|
| 48 |
+
),
|
| 49 |
+
(
|
| 50 |
+
"Beatriz, content, 4/4",
|
| 51 |
+
"Write a lullaby for: Beatriz, age 4\n"
|
| 52 |
+
"Loves: the cat next door, songs about the sea\n"
|
| 53 |
+
"Mood: cosy and content\n"
|
| 54 |
+
"Key: D major\n"
|
| 55 |
+
"Meter: 4/4",
|
| 56 |
+
),
|
| 57 |
+
]
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def parse_lullaby(text):
|
| 61 |
+
"""Same parser as the app β returns dict or raises ValueError."""
|
| 62 |
+
tempo_match = TEMPO_RE.search(text)
|
| 63 |
+
if not tempo_match:
|
| 64 |
+
raise ValueError("missing 'Tempo:' line")
|
| 65 |
+
prog_match = PROG_RE.search(text)
|
| 66 |
+
if not prog_match:
|
| 67 |
+
raise ValueError("missing 'Progression:' line")
|
| 68 |
+
|
| 69 |
+
progression = [c.strip() for c in re.split(r"\s*-\s*", prog_match.group(1)) if c.strip()]
|
| 70 |
+
|
| 71 |
+
lines = []
|
| 72 |
+
for raw in text.splitlines():
|
| 73 |
+
raw = raw.strip()
|
| 74 |
+
if not raw.startswith("["):
|
| 75 |
+
continue
|
| 76 |
+
fragments = CHORD_RE.findall(raw)
|
| 77 |
+
if fragments:
|
| 78 |
+
lines.append(fragments)
|
| 79 |
+
|
| 80 |
+
if not lines:
|
| 81 |
+
raise ValueError("no chord-marked lyric lines")
|
| 82 |
+
|
| 83 |
+
return {
|
| 84 |
+
"bpm": int(tempo_match.group(1)),
|
| 85 |
+
"meter": f"{tempo_match.group(2)}/{tempo_match.group(3)}",
|
| 86 |
+
"progression": progression,
|
| 87 |
+
"lines": lines,
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def check(label, cond, detail=""):
|
| 92 |
+
mark = "β" if cond else "β"
|
| 93 |
+
print(f" {mark} {label}" + (f" β {detail}" if detail else ""))
|
| 94 |
+
return cond
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def main():
|
| 98 |
+
if not os.path.exists(MODEL_PATH):
|
| 99 |
+
print(f"ERROR: model not found at {MODEL_PATH}")
|
| 100 |
+
print("Set LULLABY_GGUF env var or put lullaby.gguf in cwd.")
|
| 101 |
+
sys.exit(1)
|
| 102 |
+
|
| 103 |
+
print(f"Loading {MODEL_PATH}...")
|
| 104 |
+
t0 = time.time()
|
| 105 |
+
llm = Llama(
|
| 106 |
+
model_path=MODEL_PATH,
|
| 107 |
+
n_ctx=1024,
|
| 108 |
+
n_threads=4,
|
| 109 |
+
n_gpu_layers=0,
|
| 110 |
+
chat_format="llama-3",
|
| 111 |
+
verbose=False,
|
| 112 |
+
)
|
| 113 |
+
print(f"Loaded in {time.time() - t0:.1f}s\n")
|
| 114 |
+
|
| 115 |
+
total_pass = 0
|
| 116 |
+
total_checks = 0
|
| 117 |
+
|
| 118 |
+
for label, prompt in TEST_PROMPTS:
|
| 119 |
+
print(f"=== {label} ===")
|
| 120 |
+
print(f"Prompt:\n{prompt}\n")
|
| 121 |
+
t0 = time.time()
|
| 122 |
+
resp = llm.create_chat_completion(
|
| 123 |
+
messages=[
|
| 124 |
+
{"role": "system", "content": SYSTEM_PROMPT},
|
| 125 |
+
{"role": "user", "content": prompt},
|
| 126 |
+
],
|
| 127 |
+
temperature=0.75,
|
| 128 |
+
max_tokens=400,
|
| 129 |
+
top_p=0.9,
|
| 130 |
+
)
|
| 131 |
+
gen_time = time.time() - t0
|
| 132 |
+
output = resp["choices"][0]["message"]["content"].strip()
|
| 133 |
+
print(f"Output ({gen_time:.1f}s):\n{output}\n")
|
| 134 |
+
|
| 135 |
+
# Parse it
|
| 136 |
+
try:
|
| 137 |
+
parsed = parse_lullaby(output)
|
| 138 |
+
total_pass += check("parses cleanly", True)
|
| 139 |
+
total_pass += check("tempo found", "bpm" in parsed, f"{parsed['bpm']} bpm")
|
| 140 |
+
total_pass += check("progression has β₯3 chords",
|
| 141 |
+
len(parsed["progression"]) >= 3,
|
| 142 |
+
str(parsed["progression"]))
|
| 143 |
+
total_pass += check("β₯6 chord-marked lines",
|
| 144 |
+
len(parsed["lines"]) >= 6,
|
| 145 |
+
f"{len(parsed['lines'])} lines")
|
| 146 |
+
|
| 147 |
+
# Name check
|
| 148 |
+
name = prompt.split("for:")[1].split(",")[0].strip()
|
| 149 |
+
found = name.lower() in output.lower()
|
| 150 |
+
total_pass += check(f"name '{name}' in output", found)
|
| 151 |
+
total_checks += 5
|
| 152 |
+
except ValueError as e:
|
| 153 |
+
check("parses cleanly", False, str(e))
|
| 154 |
+
total_checks += 5
|
| 155 |
+
|
| 156 |
+
print()
|
| 157 |
+
|
| 158 |
+
print(f"\n=== {total_pass}/{total_checks} checks passed ===")
|
| 159 |
+
sys.exit(0 if total_pass == total_checks else 1)
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
if __name__ == "__main__":
|
| 163 |
+
main()
|
tests/test_voice.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Test voice.py independently β generates a WAV file for sample lyrics
|
| 3 |
+
in each soft voice. No LLM, no Gradio.
|
| 4 |
+
|
| 5 |
+
Usage:
|
| 6 |
+
python test_voice.py # test all soft voices
|
| 7 |
+
python test_voice.py af_kore # test only the given voice id(s)
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import sys
|
| 11 |
+
import time
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
|
| 14 |
+
import soundfile as sf
|
| 15 |
+
|
| 16 |
+
from voice import speak_lyrics, SR_TARGET, SOFT_VOICES
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
SAMPLE_LYRICS = """Mia, dear Mia,
|
| 20 |
+
the dark wraps the earth now,
|
| 21 |
+
the cat curls up near you,
|
| 22 |
+
the moon hums a soft tune...
|
| 23 |
+
|
| 24 |
+
The stars know your name now,
|
| 25 |
+
they call from above,
|
| 26 |
+
your blanket is gentle,
|
| 27 |
+
sewn warm from our love..."""
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def check_audio(audio, label):
|
| 31 |
+
"""Sanity-check the returned audio."""
|
| 32 |
+
issues = []
|
| 33 |
+
if audio is None:
|
| 34 |
+
issues.append("audio is None")
|
| 35 |
+
return issues
|
| 36 |
+
if len(audio) == 0:
|
| 37 |
+
issues.append("audio is empty")
|
| 38 |
+
return issues
|
| 39 |
+
|
| 40 |
+
duration = len(audio) / SR_TARGET
|
| 41 |
+
peak = float(max(abs(audio.min()), abs(audio.max())))
|
| 42 |
+
rms = float((audio ** 2).mean() ** 0.5)
|
| 43 |
+
|
| 44 |
+
if duration < 1.0:
|
| 45 |
+
issues.append(f"too short ({duration:.2f}s)")
|
| 46 |
+
if peak < 0.001:
|
| 47 |
+
issues.append(f"silent (peak {peak:.4f})")
|
| 48 |
+
if peak > 1.0:
|
| 49 |
+
issues.append(f"clipping (peak {peak:.4f})")
|
| 50 |
+
if rms < 0.001:
|
| 51 |
+
issues.append(f"near-silent rms ({rms:.4f})")
|
| 52 |
+
|
| 53 |
+
print(f" duration: {duration:.2f}s peak: {peak:.3f} rms: {rms:.3f}")
|
| 54 |
+
return issues
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def main():
|
| 58 |
+
out_dir = Path("voice_test_outputs")
|
| 59 |
+
out_dir.mkdir(exist_ok=True)
|
| 60 |
+
|
| 61 |
+
# Optional: restrict to specific voice ids passed on the command line.
|
| 62 |
+
requested = sys.argv[1:]
|
| 63 |
+
if requested:
|
| 64 |
+
voices = {v: SOFT_VOICES.get(v, v) for v in requested}
|
| 65 |
+
else:
|
| 66 |
+
voices = dict(SOFT_VOICES)
|
| 67 |
+
|
| 68 |
+
print("Testing voice.py with sample lullaby lyrics.\n")
|
| 69 |
+
print(f"Lyrics:\n{SAMPLE_LYRICS}\n")
|
| 70 |
+
print(f"Voices to test: {', '.join(voices.keys())}\n")
|
| 71 |
+
|
| 72 |
+
all_good = True
|
| 73 |
+
for voice_id, label in voices.items():
|
| 74 |
+
print(f"=== voice='{voice_id}' ({label}) ===")
|
| 75 |
+
t0 = time.time()
|
| 76 |
+
audio = speak_lyrics(
|
| 77 |
+
SAMPLE_LYRICS,
|
| 78 |
+
target_seconds=20.0,
|
| 79 |
+
voice=voice_id,
|
| 80 |
+
)
|
| 81 |
+
gen_time = time.time() - t0
|
| 82 |
+
print(f" generated in {gen_time:.1f}s")
|
| 83 |
+
|
| 84 |
+
issues = check_audio(audio, voice_id)
|
| 85 |
+
if issues:
|
| 86 |
+
print(f" \u2717 ISSUES: {', '.join(issues)}")
|
| 87 |
+
all_good = False
|
| 88 |
+
else:
|
| 89 |
+
print(f" \u2713 looks good")
|
| 90 |
+
out_path = out_dir / f"voice_{voice_id}.wav"
|
| 91 |
+
sf.write(str(out_path), audio, SR_TARGET, subtype="PCM_16")
|
| 92 |
+
print(f" saved \u2192 {out_path}")
|
| 93 |
+
print()
|
| 94 |
+
|
| 95 |
+
print(f"\n=== {'all voices OK' if all_good else 'some voices failed'} ===")
|
| 96 |
+
print(f"WAV files in: {out_dir.resolve()}")
|
| 97 |
+
print("\nListen to each and trim SOFT_VOICES in voice.py to the ones you like.")
|
| 98 |
+
sys.exit(0 if all_good else 1)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
if __name__ == "__main__":
|
| 102 |
+
main()
|
train/generate_dataset.py
ADDED
|
@@ -0,0 +1,599 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
generate_dataset.py β build a HIGH-VARIETY lullaby fine-tuning dataset by
|
| 3 |
+
distilling from a strong LLM (Anthropic Claude), with a hard anti-repetition
|
| 4 |
+
gate so the result can NEVER collapse into boilerplate.
|
| 5 |
+
|
| 6 |
+
WHY THIS EXISTS
|
| 7 |
+
---------------
|
| 8 |
+
The previous dataset was ~88% boilerplate: ~123 stock lines recombined across
|
| 9 |
+
2,238 "examples" (only 12% of lines were unique). A model fine-tuned on that
|
| 10 |
+
learns to reassemble a fixed kit of phrases and copies glitches like
|
| 11 |
+
"dreamings" β which is exactly what we saw in both the 3B and 8B outputs.
|
| 12 |
+
|
| 13 |
+
The fix is to DISTILL from a model that already writes varied, coherent
|
| 14 |
+
lullabies, and to enforce variety mechanically:
|
| 15 |
+
|
| 16 |
+
1. Diverse prompt specs β large pools of names (many cultures), concrete and
|
| 17 |
+
unusual "loves", varied fears/moods/ages, all keys + meters β sampled so
|
| 18 |
+
combinations essentially never repeat.
|
| 19 |
+
2. A strong LLM writes each lullaby fresh at a creativity-friendly
|
| 20 |
+
temperature, in your EXACT output format.
|
| 21 |
+
3. DEDUP GATE: every generated lyric line is normalized and checked against
|
| 22 |
+
all lines already accepted. If too many lines overlap with existing
|
| 23 |
+
content (n-gram Jaccard), the example is rejected and regenerated. This
|
| 24 |
+
structurally prevents the boilerplate failure mode.
|
| 25 |
+
4. FORMAT GATE: each completion must parse β tempo header, [chord] markers,
|
| 26 |
+
a sensible line count β or it's regenerated.
|
| 27 |
+
|
| 28 |
+
USAGE
|
| 29 |
+
-----
|
| 30 |
+
export ANTHROPIC_API_KEY=sk-ant-...
|
| 31 |
+
python generate_dataset.py --n 1500 --out lullaby_dataset.jsonl
|
| 32 |
+
|
| 33 |
+
# resume/append to an existing file (keeps its lines in the dedup index):
|
| 34 |
+
python generate_dataset.py --n 500 --out lullaby_dataset.jsonl --append
|
| 35 |
+
|
| 36 |
+
Notes
|
| 37 |
+
-----
|
| 38 |
+
- Costs real API tokens. 1,500 short completions on a small/cheap model is
|
| 39 |
+
inexpensive; check your account.
|
| 40 |
+
- A clean 1,000-1,500 examples on the FAST 3B beats a dirty 2,238 on any size
|
| 41 |
+
model β you do NOT need the 8B once the data is good.
|
| 42 |
+
"""
|
| 43 |
+
|
| 44 |
+
import argparse
|
| 45 |
+
import json
|
| 46 |
+
import os
|
| 47 |
+
import random
|
| 48 |
+
import re
|
| 49 |
+
import sys
|
| 50 |
+
import time
|
| 51 |
+
|
| 52 |
+
# ----------------------------------------------------------------------
|
| 53 |
+
# Prompt-spec pools β broad and concrete, to drive variety at the source.
|
| 54 |
+
# ----------------------------------------------------------------------
|
| 55 |
+
NAMES = [
|
| 56 |
+
# deliberately multicultural so the model doesn't tie lullabies to one
|
| 57 |
+
# naming style
|
| 58 |
+
"Mia", "Theo", "Aiko", "Luca", "Noa", "Amara", "Kai", "Sofia", "Idris",
|
| 59 |
+
"Lena", "Mateo", "Yuki", "Omar", "Freya", "Diego", "Priya", "Finn",
|
| 60 |
+
"Zara", "Hana", "Niko", "Maya", "Arlo", "Ines", "Tariq", "Lila", "Oskar",
|
| 61 |
+
"Nadia", "Bo", "Esme", "Rafael", "Suri", "Milo", "Anya", "Jonah", "Leilani",
|
| 62 |
+
"Cyrus", "Wren", "Ravi", "Talia", "Soren", "Beatriz", "Γmile", "Saoirse",
|
| 63 |
+
"Junie", "Cosmo", "Indra", "Pax", "Nour", "Eli", "Mei",
|
| 64 |
+
]
|
| 65 |
+
# DIVERSITY: a fixed list of specific loves clusters themes; but a category
|
| 66 |
+
# with parenthetical *examples* also clusters, because the LLM anchors on the
|
| 67 |
+
# first/easiest example we list (we saw fireflies in 20% and blankets in 25%
|
| 68 |
+
# of a 20-sample test). The fix: each category carries a POOL of concrete
|
| 69 |
+
# sub-topics, and make_spec() samples a SPECIFIC sub-topic to steer the LLM
|
| 70 |
+
# toward β so it's pushed to "a hot-air balloon" or "a tide pool", not the
|
| 71 |
+
# obvious "fireflies" every time. Combined with the theme-cap in main(), this
|
| 72 |
+
# spreads coverage widely.
|
| 73 |
+
LOVE_CATEGORIES = {
|
| 74 |
+
"animal/pet": [
|
| 75 |
+
"a backyard hedgehog", "a pair of ducks on the pond", "a gentle old horse",
|
| 76 |
+
"a tabby cat who naps in the sun", "a green tree frog", "a family goat",
|
| 77 |
+
"a snail with a swirly shell", "a robin that visits the windowsill",
|
| 78 |
+
"a stuffed fox named after a color", "a tortoise who moves slowly",
|
| 79 |
+
"a pony with a braided mane", "a little brown bat at dusk",
|
| 80 |
+
"a goldfish circling its bowl", "a sheepdog who herds the chickens",
|
| 81 |
+
"a ladybug on a leaf", "a stuffed whale", "a chubby garden bumblebee",
|
| 82 |
+
],
|
| 83 |
+
"vehicle/machine": [
|
| 84 |
+
"a red tugboat in the harbor", "a hot-air balloon drifting up",
|
| 85 |
+
"a yellow digger at the building site", "the night train through the valley",
|
| 86 |
+
"a little wooden sailboat", "a cable car climbing the hill",
|
| 87 |
+
"a tractor in the far field", "a propeller plane doing loops",
|
| 88 |
+
"a windmill turning slowly", "a fire engine (quiet, parked for the night)",
|
| 89 |
+
"a paddle steamer on the river", "a kite shaped like a dragon",
|
| 90 |
+
],
|
| 91 |
+
"nature": [
|
| 92 |
+
"the first snow settling on the fence", "a tide pool full of tiny crabs",
|
| 93 |
+
"dandelion seeds on the wind", "a mossy log in the woods",
|
| 94 |
+
"the smell of rain on warm stones", "a field of swaying barley",
|
| 95 |
+
"an acorn that might become a tree", "the frost patterns on the window",
|
| 96 |
+
"a rainbow after the storm", "pinecones under the big fir",
|
| 97 |
+
"the harvest moon low and orange", "a stream over smooth pebbles",
|
| 98 |
+
"autumn leaves spinning down",
|
| 99 |
+
],
|
| 100 |
+
"comfort object": [
|
| 101 |
+
"a well-loved cardboard-box spaceship", "a knitted octopus with long legs",
|
| 102 |
+
"a flashlight for reading under the covers", "a tiny brass music box",
|
| 103 |
+
"a worn picture book read every night", "a pillow fort in the corner",
|
| 104 |
+
"a pair of fuzzy dinosaur slippers", "a smooth worry-stone from the beach",
|
| 105 |
+
"a paper lantern shaped like a moon", "a hand-me-down rocking chair",
|
| 106 |
+
],
|
| 107 |
+
"activity": [
|
| 108 |
+
"stomping in rain puddles in red boots", "folding paper into cranes",
|
| 109 |
+
"skipping stones across the lake", "planting seeds in little pots",
|
| 110 |
+
"blowing soap bubbles in the yard", "spinning until dizzy on the grass",
|
| 111 |
+
"kneading dough beside a grandparent", "drawing chalk roads on the path",
|
| 112 |
+
"collecting shells along the shore", "flying down the slide at the park",
|
| 113 |
+
"humming into a paper-towel tube", "catching snowflakes on the tongue",
|
| 114 |
+
],
|
| 115 |
+
"place": [
|
| 116 |
+
"the treehouse with the rope ladder", "grandpa's greenhouse full of tomatoes",
|
| 117 |
+
"the little library with the bean-bag corner", "the lighthouse on the point",
|
| 118 |
+
"the orchard where apples drop", "a blanket fort under the dining table",
|
| 119 |
+
"the tide line where the sand is wet", "the attic with the round window",
|
| 120 |
+
"the meadow behind the school", "the corner bakery at dawn",
|
| 121 |
+
],
|
| 122 |
+
"person": [
|
| 123 |
+
"a big sister who tells bedtime stories", "grandpa's slow accordion tunes",
|
| 124 |
+
"an auntie who whistles while she cooks", "a baby brother's tiny yawn",
|
| 125 |
+
"a grandmother braiding hair on her lap", "a dad who does silly voices",
|
| 126 |
+
"the neighbor who shares garden strawberries", "a cousin who builds dens",
|
| 127 |
+
],
|
| 128 |
+
"tiny wonder": [
|
| 129 |
+
"the echo in the stairwell", "shadows the curtain makes on the wall",
|
| 130 |
+
"steam curling off warm milk", "the squeak of fresh snow underfoot",
|
| 131 |
+
"raindrops racing down the window", "the click of the ceiling fan",
|
| 132 |
+
"dust motes floating in a sunbeam", "the way a spoon clinks in a cup",
|
| 133 |
+
"ripples spreading when a pebble drops",
|
| 134 |
+
],
|
| 135 |
+
"sound": [
|
| 136 |
+
"the foghorn far out on the water", "rain drumming on a tin roof",
|
| 137 |
+
"crickets in the long summer grass", "a distant church bell at evening",
|
| 138 |
+
"the purr of a sleeping cat", "wind humming through the gate",
|
| 139 |
+
"the tick of grandfather's clock", "a mourning dove at first light",
|
| 140 |
+
],
|
| 141 |
+
"food/treat": [
|
| 142 |
+
"porridge with a swirl of honey", "warm pear slices after supper",
|
| 143 |
+
"a single square of dark chocolate", "fresh blackberries from the bramble",
|
| 144 |
+
"buttered crumpets on a rainy day", "a cup of warm milk with nutmeg",
|
| 145 |
+
"grandma's lemon biscuits", "roasted chestnuts in winter",
|
| 146 |
+
],
|
| 147 |
+
}
|
| 148 |
+
|
| 149 |
+
FEAR_TOPICS = [
|
| 150 |
+
"the dark when the light goes out",
|
| 151 |
+
"long shadows the wardrobe makes",
|
| 152 |
+
"thunder rumbling far away", "the howl of the wind at the window",
|
| 153 |
+
"fireworks banging on a holiday night",
|
| 154 |
+
"a monster they imagine under the bed",
|
| 155 |
+
"a creak in the closet at night",
|
| 156 |
+
"a parent leaving the room at bedtime",
|
| 157 |
+
"sleeping alone for the first time",
|
| 158 |
+
"the big dog barking behind the fence",
|
| 159 |
+
"the deep end of the swimming pool",
|
| 160 |
+
"high places and looking down",
|
| 161 |
+
"a bad dream about getting lost",
|
| 162 |
+
"waking and not finding anyone near",
|
| 163 |
+
None, None, None, # ~a quarter have no fear
|
| 164 |
+
]
|
| 165 |
+
MOODS = [
|
| 166 |
+
"sleepy and comforted", "restless but settling", "tearful, needs soothing",
|
| 167 |
+
"cosy and content", "overtired and fussy", "calm and drowsy",
|
| 168 |
+
"wired after a long day", "missing a parent who's away",
|
| 169 |
+
"scared after a bad dream", "peaceful and almost asleep",
|
| 170 |
+
"excited but winding down", "clingy and needing reassurance",
|
| 171 |
+
]
|
| 172 |
+
KEYS = [
|
| 173 |
+
"C major", "G major", "D major", "A major", "E major", "F major",
|
| 174 |
+
"Bb major", "A minor", "E minor", "D minor", "B minor", "G minor",
|
| 175 |
+
]
|
| 176 |
+
METERS = ["6/8", "3/4", "4/4"]
|
| 177 |
+
AGES = [1, 2, 2, 3, 3, 3, 4, 4, 5, 5, 6, 7]
|
| 178 |
+
|
| 179 |
+
# Chord progressions keyed by tonic, for the format guidance (the LLM picks,
|
| 180 |
+
# but we suggest diatonic options so the chord markers stay sensible).
|
| 181 |
+
PROGRESSIONS = {
|
| 182 |
+
"C major": ["C - Am - F - G", "C - G - Am - F", "C - F - G - C", "C - Em - F - G"],
|
| 183 |
+
"G major": ["G - Em - C - D", "G - D - Em - C", "G - C - D - G"],
|
| 184 |
+
"D major": ["D - Bm - G - A", "D - A - Bm - G", "D - G - A - D"],
|
| 185 |
+
"A major": ["A - F#m - D - E", "A - E - F#m - D"],
|
| 186 |
+
"E major": ["E - C#m - A - B", "E - B - C#m - A"],
|
| 187 |
+
"F major": ["F - Dm - Bb - C", "F - C - Dm - Bb"],
|
| 188 |
+
"Bb major": ["Bb - Gm - Eb - F", "Bb - F - Gm - Eb"],
|
| 189 |
+
"A minor": ["Am - G - F - G", "Am - F - C - G", "Am - Dm - E - Am"],
|
| 190 |
+
"E minor": ["Em - C - G - D", "Em - Am - B7 - Em"],
|
| 191 |
+
"D minor": ["Dm - Bb - F - C", "Dm - Gm - A - Dm"],
|
| 192 |
+
"B minor": ["Bm - G - D - A", "Bm - Em - F#m - Bm"],
|
| 193 |
+
"G minor": ["Gm - Eb - Bb - F", "Gm - Cm - D - Gm"],
|
| 194 |
+
}
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def make_spec():
|
| 198 |
+
"""Sample one diverse prompt specification. We pick a love CATEGORY, then
|
| 199 |
+
a SPECIFIC sub-topic within it to steer the teacher away from clichΓ©s,
|
| 200 |
+
and record the category as a `theme` so main() can cap over-used themes."""
|
| 201 |
+
key = random.choice(KEYS)
|
| 202 |
+
love_cat = random.choice(list(LOVE_CATEGORIES.keys()))
|
| 203 |
+
love_topic = random.choice(LOVE_CATEGORIES[love_cat])
|
| 204 |
+
return {
|
| 205 |
+
"name": random.choice(NAMES),
|
| 206 |
+
"age": random.choice(AGES),
|
| 207 |
+
"theme": love_cat, # broad theme, for the theme-cap
|
| 208 |
+
"love_topic": love_topic, # specific steer for the teacher
|
| 209 |
+
"fear_topic": random.choice(FEAR_TOPICS),
|
| 210 |
+
"mood": random.choice(MOODS),
|
| 211 |
+
"key": key,
|
| 212 |
+
"meter": random.choice(METERS),
|
| 213 |
+
"progression": random.choice(PROGRESSIONS[key]),
|
| 214 |
+
}
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def build_prompt(spec, love, fear):
|
| 218 |
+
"""Build the STORED training prompt from the concrete love/fear the
|
| 219 |
+
teacher invented β so each training example looks exactly like a real
|
| 220 |
+
user request (matching app.py's build_prompt format)."""
|
| 221 |
+
lines = [f"Write a lullaby for: {spec['name']}, age {spec['age']}"]
|
| 222 |
+
if love:
|
| 223 |
+
lines.append(f"Loves: {love}")
|
| 224 |
+
if fear:
|
| 225 |
+
lines.append(f"Fears: {fear}")
|
| 226 |
+
lines.append(f"Mood: {spec['mood']}")
|
| 227 |
+
lines.append(f"Key: {spec['key']}")
|
| 228 |
+
lines.append(f"Meter: {spec['meter']}")
|
| 229 |
+
return "\n".join(lines)
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
# ----------------------------------------------------------------------
|
| 233 |
+
# The instruction we give the strong LLM (the "teacher").
|
| 234 |
+
# ----------------------------------------------------------------------
|
| 235 |
+
TEACHER_SYSTEM = (
|
| 236 |
+
"You are a gifted children's songwriter. You write short, original, "
|
| 237 |
+
"soothing lullabies. Every line must be grammatical and make literal "
|
| 238 |
+
"sense. Weave the child's loves and fears in NATURALLY β show a loved "
|
| 239 |
+
"thing doing what it really does (a dog curls up beside them; a train "
|
| 240 |
+
"slows for the night), and gently soothe fears away. Never force a thing "
|
| 241 |
+
"into a metaphor that doesn't make sense.\n"
|
| 242 |
+
"VARIETY IS CRITICAL. Use FRESH imagery and FRESH sentence shapes every "
|
| 243 |
+
"time. Do NOT reuse stock openers like '<Name>'s eyes are growing soft' "
|
| 244 |
+
"or '<Name> watches the ___ glow', and do NOT default to fireflies, "
|
| 245 |
+
"embroidered blankets, or twinkling stars unless explicitly asked β vary "
|
| 246 |
+
"your verbs, your line structure, and your images. Each lullaby should "
|
| 247 |
+
"feel written from scratch for THIS child.\n"
|
| 248 |
+
"SAFETY: this is for a small child at bedtime. Anything you invent must "
|
| 249 |
+
"be wholesome and age-appropriate. Never involve death, violence, "
|
| 250 |
+
"weapons, horror, substances, or anything frightening beyond a gentle, "
|
| 251 |
+
"easily-soothed childhood worry."
|
| 252 |
+
)
|
| 253 |
+
|
| 254 |
+
TEACHER_FORMAT = """You are given a specific LOVE (and maybe a FEAR). Use them
|
| 255 |
+
as the heart of the lullaby β but render them in your own fresh words. Echo
|
| 256 |
+
the given love faithfully (don't swap it for a different thing), then write
|
| 257 |
+
the lullaby weaving it in.
|
| 258 |
+
|
| 259 |
+
Output EXACTLY this, and nothing else:
|
| 260 |
+
|
| 261 |
+
LOVE: <restate the given love in a natural phrase a parent might type>
|
| 262 |
+
FEAR: <the given fear in a natural phrase, or "none">
|
| 263 |
+
Tempo: <40-70>bpm, <meter>
|
| 264 |
+
Progression: <the given progression>
|
| 265 |
+
|
| 266 |
+
[Chord] lyric words [Chord] more words,
|
| 267 |
+
[Chord] next line [Chord] continuing,
|
| 268 |
+
[Chord] third line [Chord] of verse,
|
| 269 |
+
[Chord] fourth line [Chord] ending...
|
| 270 |
+
|
| 271 |
+
[Chord] second verse [Chord] line one,
|
| 272 |
+
[Chord] line two [Chord] continuing,
|
| 273 |
+
[Chord] line three [Chord] of verse,
|
| 274 |
+
[Chord] final line [Chord] ending...
|
| 275 |
+
|
| 276 |
+
Rules:
|
| 277 |
+
- The LOVE and FEAR lines come FIRST, then Tempo, then Progression, then a
|
| 278 |
+
blank line, then the lullaby.
|
| 279 |
+
- Exactly two 4-line verses (8 lyric lines total).
|
| 280 |
+
- Put chord markers in [square brackets] inline before the words they cover,
|
| 281 |
+
using ONLY chords from the given progression.
|
| 282 |
+
- The child's name must appear naturally in the lyric.
|
| 283 |
+
- Keep lines short and singable (3-7 words).
|
| 284 |
+
- VARY your opening line β do NOT start with "<Name>'s eyes are growing
|
| 285 |
+
soft" or "<Name> watches...". Find a different way in each time.
|
| 286 |
+
- Everything must be wholesome and age-appropriate.
|
| 287 |
+
- End the lullaby softly (trailing "..." is fine)."""
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
def teacher_user(spec):
|
| 291 |
+
fear_line = (f"FEAR (soothe this gently): {spec['fear_topic']}"
|
| 292 |
+
if spec["fear_topic"] else "FEAR: none")
|
| 293 |
+
return (
|
| 294 |
+
f"Write a lullaby for: {spec['name']}, age {spec['age']}\n"
|
| 295 |
+
f"LOVE (make this the heart of the song): {spec['love_topic']}\n"
|
| 296 |
+
f"{fear_line}\n"
|
| 297 |
+
f"Mood: {spec['mood']}\n"
|
| 298 |
+
f"Key: {spec['key']}\n"
|
| 299 |
+
f"Meter: {spec['meter']}\n"
|
| 300 |
+
f"Use this progression: {spec['progression']}\n\n"
|
| 301 |
+
+ TEACHER_FORMAT
|
| 302 |
+
)
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
# ----------------------------------------------------------------------
|
| 306 |
+
# LLM call (Anthropic). Swap here if you prefer another provider.
|
| 307 |
+
# ----------------------------------------------------------------------
|
| 308 |
+
def call_teacher(spec, model, temperature):
|
| 309 |
+
import anthropic
|
| 310 |
+
client = anthropic.Anthropic() # reads ANTHROPIC_API_KEY
|
| 311 |
+
msg = client.messages.create(
|
| 312 |
+
model=model,
|
| 313 |
+
max_tokens=600,
|
| 314 |
+
temperature=temperature,
|
| 315 |
+
system=TEACHER_SYSTEM,
|
| 316 |
+
messages=[{"role": "user", "content": teacher_user(spec)}],
|
| 317 |
+
)
|
| 318 |
+
return "".join(b.text for b in msg.content if getattr(b, "type", "") == "text").strip()
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def split_teacher_output(raw):
|
| 322 |
+
"""Pull the declared LOVE/FEAR header off the teacher output.
|
| 323 |
+
Returns (love, fear_or_None, completion_without_love_fear_lines)."""
|
| 324 |
+
love, fear = None, None
|
| 325 |
+
body_lines = []
|
| 326 |
+
for line in raw.split("\n"):
|
| 327 |
+
s = line.strip()
|
| 328 |
+
if love is None and s.upper().startswith("LOVE:"):
|
| 329 |
+
love = s.split(":", 1)[1].strip()
|
| 330 |
+
continue
|
| 331 |
+
if fear is None and s.upper().startswith("FEAR:"):
|
| 332 |
+
f = s.split(":", 1)[1].strip()
|
| 333 |
+
fear = None if f.lower() in ("none", "no", "n/a", "") else f
|
| 334 |
+
continue
|
| 335 |
+
body_lines.append(line)
|
| 336 |
+
# strip leading blank lines from the body
|
| 337 |
+
while body_lines and not body_lines[0].strip():
|
| 338 |
+
body_lines.pop(0)
|
| 339 |
+
return love, fear, "\n".join(body_lines).strip()
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
# ----------------------------------------------------------------------
|
| 343 |
+
# Validation + dedup
|
| 344 |
+
# ----------------------------------------------------------------------
|
| 345 |
+
def lyric_lines(completion):
|
| 346 |
+
out = []
|
| 347 |
+
for line in completion.split("\n"):
|
| 348 |
+
line = line.strip()
|
| 349 |
+
if not line or line.startswith("Tempo:") or line.startswith("Progression:"):
|
| 350 |
+
continue
|
| 351 |
+
clean = re.sub(r"\[[^\]]*\]", "", line).strip().lower()
|
| 352 |
+
clean = re.sub(r"[^\w\s]", "", clean)
|
| 353 |
+
clean = re.sub(r"\s+", " ", clean).strip()
|
| 354 |
+
if clean:
|
| 355 |
+
out.append(clean)
|
| 356 |
+
return out
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
def format_ok(completion, spec):
|
| 360 |
+
if not completion.startswith("Tempo:"):
|
| 361 |
+
return False
|
| 362 |
+
if "Progression:" not in completion:
|
| 363 |
+
return False
|
| 364 |
+
if "[" not in completion or "]" not in completion:
|
| 365 |
+
return False
|
| 366 |
+
lines = lyric_lines(completion)
|
| 367 |
+
if not (6 <= len(lines) <= 10):
|
| 368 |
+
return False
|
| 369 |
+
# name must appear
|
| 370 |
+
if spec["name"].lower() not in completion.lower():
|
| 371 |
+
return False
|
| 372 |
+
# chords used in [brackets] must all be valid synth chords AND must be
|
| 373 |
+
# drawn from the progression. Exact-match each bracket token (no loose
|
| 374 |
+
# regex that could mis-split e.g. "Bbmaj7" into "Bbm").
|
| 375 |
+
prog_chords = _parse_progression_chords(spec["progression"])
|
| 376 |
+
if not prog_chords:
|
| 377 |
+
return False
|
| 378 |
+
prog_set = set(prog_chords)
|
| 379 |
+
used = {u.strip() for u in re.findall(r"\[([^\]]*)\]", completion)}
|
| 380 |
+
used.discard("") # allow empty [] just in case, though we don't emit them
|
| 381 |
+
if not used:
|
| 382 |
+
return False
|
| 383 |
+
for ch in used:
|
| 384 |
+
if ch not in VALID_CHORDS: # not a chord the synths understand
|
| 385 |
+
return False
|
| 386 |
+
if ch not in prog_set: # not from the requested progression
|
| 387 |
+
return False
|
| 388 |
+
return True
|
| 389 |
+
|
| 390 |
+
|
| 391 |
+
# Valid chords MUST match what the synths actually accept. The synths parse a
|
| 392 |
+
# chord as <root><quality> where root in NOTE_TO_SEMI and quality in
|
| 393 |
+
# CHORD_INTERVALS (synths/guitar.py, musicbox.py, piano.py all share these).
|
| 394 |
+
# We build the exact set so the validator and the synths can never drift β
|
| 395 |
+
# and so we don't mis-split compound chords (the old loose regex turned
|
| 396 |
+
# "Bbmaj7" into "Bbm").
|
| 397 |
+
_ROOTS = ["C", "C#", "Db", "D", "D#", "Eb", "E", "F",
|
| 398 |
+
"F#", "Gb", "G", "G#", "Ab", "A", "A#", "Bb", "B"]
|
| 399 |
+
_QUALITIES = ["", "m", "7", "m7", "maj7", "sus2", "sus4"]
|
| 400 |
+
VALID_CHORDS = {r + q for r in _ROOTS for q in _QUALITIES}
|
| 401 |
+
|
| 402 |
+
|
| 403 |
+
def _parse_progression_chords(progression):
|
| 404 |
+
"""Split 'Bb - Gm - Eb - F' into ['Bb','Gm','Eb','F'] using exact chord
|
| 405 |
+
matching (NOT a loose regex). Returns None if any token isn't a valid
|
| 406 |
+
synth chord."""
|
| 407 |
+
chords = []
|
| 408 |
+
for t in (tok.strip() for tok in progression.split("-")):
|
| 409 |
+
if t in VALID_CHORDS:
|
| 410 |
+
chords.append(t)
|
| 411 |
+
else:
|
| 412 |
+
return None
|
| 413 |
+
return chords
|
| 414 |
+
|
| 415 |
+
|
| 416 |
+
def _ngrams(line, n=3):
|
| 417 |
+
toks = line.split()
|
| 418 |
+
if len(toks) < n:
|
| 419 |
+
return {line}
|
| 420 |
+
return {" ".join(toks[i:i+n]) for i in range(len(toks) - n + 1)}
|
| 421 |
+
|
| 422 |
+
|
| 423 |
+
class DedupIndex:
|
| 424 |
+
"""Tracks all accepted lyric lines (as 3-gram sets) and rejects new
|
| 425 |
+
examples that overlap too much with existing content."""
|
| 426 |
+
def __init__(self, max_line_overlap=0.5, max_song_dup_lines=2):
|
| 427 |
+
self.seen_ngrams = set()
|
| 428 |
+
self.seen_lines = set()
|
| 429 |
+
self.max_line_overlap = max_line_overlap
|
| 430 |
+
self.max_song_dup_lines = max_song_dup_lines
|
| 431 |
+
|
| 432 |
+
def index_existing(self, completion):
|
| 433 |
+
for ln in lyric_lines(completion):
|
| 434 |
+
self.seen_lines.add(ln)
|
| 435 |
+
self.seen_ngrams |= _ngrams(ln)
|
| 436 |
+
|
| 437 |
+
def too_repetitive(self, completion):
|
| 438 |
+
lines = lyric_lines(completion)
|
| 439 |
+
dup_lines = 0
|
| 440 |
+
for ln in lines:
|
| 441 |
+
if ln in self.seen_lines:
|
| 442 |
+
dup_lines += 1
|
| 443 |
+
continue
|
| 444 |
+
grams = _ngrams(ln)
|
| 445 |
+
if grams:
|
| 446 |
+
overlap = len(grams & self.seen_ngrams) / len(grams)
|
| 447 |
+
if overlap > self.max_line_overlap:
|
| 448 |
+
dup_lines += 1
|
| 449 |
+
return dup_lines > self.max_song_dup_lines
|
| 450 |
+
|
| 451 |
+
def add(self, completion):
|
| 452 |
+
self.index_existing(completion)
|
| 453 |
+
|
| 454 |
+
|
| 455 |
+
def main():
|
| 456 |
+
ap = argparse.ArgumentParser()
|
| 457 |
+
ap.add_argument("--n", type=int, default=1500, help="examples to generate")
|
| 458 |
+
ap.add_argument("--out", default="lullaby_dataset.jsonl")
|
| 459 |
+
ap.add_argument("--model", default="claude-3-5-haiku-20241022",
|
| 460 |
+
help="teacher model (cheap+good is ideal for distillation)")
|
| 461 |
+
ap.add_argument("--temperature", type=float, default=1.0)
|
| 462 |
+
ap.add_argument("--append", action="store_true",
|
| 463 |
+
help="append to --out and seed dedup from its contents")
|
| 464 |
+
ap.add_argument("--max-attempts", type=int, default=6,
|
| 465 |
+
help="regeneration attempts per example before skipping")
|
| 466 |
+
args = ap.parse_args()
|
| 467 |
+
|
| 468 |
+
if not os.environ.get("ANTHROPIC_API_KEY"):
|
| 469 |
+
print("ERROR: set ANTHROPIC_API_KEY", file=sys.stderr)
|
| 470 |
+
sys.exit(1)
|
| 471 |
+
|
| 472 |
+
dedup = DedupIndex()
|
| 473 |
+
mode = "a" if args.append else "w"
|
| 474 |
+
existing = 0
|
| 475 |
+
if args.append and os.path.exists(args.out):
|
| 476 |
+
for line in open(args.out):
|
| 477 |
+
try:
|
| 478 |
+
dedup.index_existing(json.loads(line)["completion"])
|
| 479 |
+
existing += 1
|
| 480 |
+
except Exception:
|
| 481 |
+
pass
|
| 482 |
+
print(f"Seeded dedup from {existing} existing examples.")
|
| 483 |
+
|
| 484 |
+
written = 0
|
| 485 |
+
rejected_fmt = 0
|
| 486 |
+
rejected_dup = 0
|
| 487 |
+
rejected_unsafe = 0
|
| 488 |
+
rejected_theme = 0
|
| 489 |
+
rejected_opener = 0
|
| 490 |
+
from collections import Counter
|
| 491 |
+
theme_counts = Counter()
|
| 492 |
+
opener_counts = Counter()
|
| 493 |
+
# No single love-theme may exceed this share of the dataset (10 themes, so
|
| 494 |
+
# an even split is 10%; we allow a little headroom at 16%).
|
| 495 |
+
theme_cap_frac = 0.16
|
| 496 |
+
t0 = time.time()
|
| 497 |
+
spins_since_write = 0
|
| 498 |
+
with open(args.out, mode) as f:
|
| 499 |
+
while written < args.n:
|
| 500 |
+
spec = make_spec()
|
| 501 |
+
|
| 502 |
+
# THEME-CAP: if this love-theme is already over its share of the
|
| 503 |
+
# dataset, skip and resample a different spec. A spin guard relaxes
|
| 504 |
+
# the cap if we can't quickly find an under-cap theme, preventing
|
| 505 |
+
# any chance of an infinite resample loop near the end of a run.
|
| 506 |
+
spins_since_write += 1
|
| 507 |
+
relax = spins_since_write > 200
|
| 508 |
+
if written >= 40 and not relax:
|
| 509 |
+
cap = max(2, int(theme_cap_frac * written))
|
| 510 |
+
if theme_counts[spec["theme"]] >= cap:
|
| 511 |
+
rejected_theme += 1
|
| 512 |
+
continue
|
| 513 |
+
|
| 514 |
+
accepted = False
|
| 515 |
+
for attempt in range(args.max_attempts):
|
| 516 |
+
# nudge temperature up slightly on retries for more variety
|
| 517 |
+
temp = min(1.0, args.temperature + 0.05 * attempt)
|
| 518 |
+
try:
|
| 519 |
+
raw = call_teacher(spec, args.model, temp)
|
| 520 |
+
except Exception as e:
|
| 521 |
+
print(f" API error: {e}; backing off", file=sys.stderr)
|
| 522 |
+
time.sleep(2 + attempt)
|
| 523 |
+
continue
|
| 524 |
+
# Split off the declared concrete LOVE/FEAR header.
|
| 525 |
+
love, fear, completion = split_teacher_output(raw)
|
| 526 |
+
# SAFETY (Fix B): even though the teacher is told to stay
|
| 527 |
+
# wholesome, double-check the invented love/fear AND the
|
| 528 |
+
# lyric body. Drop anything that slips through.
|
| 529 |
+
from safety import is_safe
|
| 530 |
+
if not (is_safe(love) and is_safe(fear) and is_safe(completion)):
|
| 531 |
+
rejected_unsafe += 1
|
| 532 |
+
continue
|
| 533 |
+
if not format_ok(completion, spec):
|
| 534 |
+
rejected_fmt += 1
|
| 535 |
+
continue
|
| 536 |
+
if dedup.too_repetitive(completion):
|
| 537 |
+
rejected_dup += 1
|
| 538 |
+
continue
|
| 539 |
+
# OPENER-DEDUP: reject if the first lyric line uses an
|
| 540 |
+
# over-used opening shape (the model's favorite stock
|
| 541 |
+
# openers like "<name> watches..." / "...eyes are growing
|
| 542 |
+
# soft"). We normalize the opener by replacing the name with
|
| 543 |
+
# <N> and keeping the first 4 words.
|
| 544 |
+
lls = lyric_lines(completion)
|
| 545 |
+
opener = ""
|
| 546 |
+
if lls:
|
| 547 |
+
words = lls[0].split()
|
| 548 |
+
opener = " ".join(words[:4])
|
| 549 |
+
opener = opener.replace(spec["name"].lower(), "<n>")
|
| 550 |
+
if opener and opener_counts[opener] >= 3:
|
| 551 |
+
rejected_opener += 1
|
| 552 |
+
continue
|
| 553 |
+
# accept β store a prompt built from the CONCRETE love/fear so
|
| 554 |
+
# the example mirrors a real user request.
|
| 555 |
+
rec = {"prompt": build_prompt(spec, love, fear),
|
| 556 |
+
"completion": completion}
|
| 557 |
+
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
| 558 |
+
f.flush()
|
| 559 |
+
dedup.add(completion)
|
| 560 |
+
theme_counts[spec["theme"]] += 1
|
| 561 |
+
if opener:
|
| 562 |
+
opener_counts[opener] += 1
|
| 563 |
+
written += 1
|
| 564 |
+
accepted = True
|
| 565 |
+
spins_since_write = 0
|
| 566 |
+
break
|
| 567 |
+
if not accepted:
|
| 568 |
+
# couldn't get a fresh, valid one for this spec β skip it
|
| 569 |
+
pass
|
| 570 |
+
if written and written % 25 == 0:
|
| 571 |
+
rate = written / (time.time() - t0)
|
| 572 |
+
print(f" {written}/{args.n} "
|
| 573 |
+
f"(fmt-rej {rejected_fmt}, dup-rej {rejected_dup}, "
|
| 574 |
+
f"{rate:.1f}/s)")
|
| 575 |
+
|
| 576 |
+
total = existing + written if args.append else written
|
| 577 |
+
print(f"\nDone. Wrote {written} new examples β {args.out} "
|
| 578 |
+
f"(total {total}).")
|
| 579 |
+
print(f"Rejected: {rejected_fmt} format, {rejected_dup} too-repetitive, "
|
| 580 |
+
f"{rejected_unsafe} unsafe, {rejected_theme} theme-cap, "
|
| 581 |
+
f"{rejected_opener} stock-opener.")
|
| 582 |
+
# Report final uniqueness
|
| 583 |
+
all_lines = []
|
| 584 |
+
for line in open(args.out):
|
| 585 |
+
all_lines += lyric_lines(json.loads(line)["completion"])
|
| 586 |
+
if all_lines:
|
| 587 |
+
uniq = len(set(all_lines)) / len(all_lines)
|
| 588 |
+
print(f"Line uniqueness: {uniq*100:.0f}% "
|
| 589 |
+
f"({len(set(all_lines))}/{len(all_lines)}) "
|
| 590 |
+
f"[old dataset was 12%]")
|
| 591 |
+
# Theme distribution (this run only)
|
| 592 |
+
if theme_counts:
|
| 593 |
+
print("Theme distribution:")
|
| 594 |
+
for th, c in theme_counts.most_common():
|
| 595 |
+
print(f" {c:4d} ({100*c/max(1,written):4.0f}%) {th}")
|
| 596 |
+
|
| 597 |
+
|
| 598 |
+
if __name__ == "__main__":
|
| 599 |
+
main()
|
train/generate_dataset_requirements.txt
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Dependencies for generate_dataset.py (dataset building only β NOT needed to
|
| 2 |
+
# run the app). Install separately:
|
| 3 |
+
# pip install -r generate_dataset_requirements.txt
|
| 4 |
+
anthropic>=0.40
|
train/train_lullaby.ipynb
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
utils/__init__.py
ADDED
|
File without changes
|
utils/safety.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
safety.py β shared content-safety screening for lullaby inputs.
|
| 3 |
+
|
| 4 |
+
Used in TWO places so the rules can never diverge:
|
| 5 |
+
- app.py (Fix A): screens the USER's "loves"/"fears" free-text at runtime,
|
| 6 |
+
so a deployed-app user can't get a lullaby built around something
|
| 7 |
+
inappropriate (death, weapons, violence, etc.). Rejects with a gentle
|
| 8 |
+
message.
|
| 9 |
+
- generate_dataset.py (Fix B): screens the loves/fears that go into each
|
| 10 |
+
training example, so no training data is built around dark themes β even
|
| 11 |
+
when the teacher LLM is allowed to invent specifics.
|
| 12 |
+
|
| 13 |
+
The screening is intentionally simple and conservative: a substring/word
|
| 14 |
+
match against a focused list of clearly-inappropriate-for-a-child's-lullaby
|
| 15 |
+
terms. It is NOT a general profanity filter or a semantic classifier β it's a
|
| 16 |
+
targeted guard for the specific failure mode "user asks for a lullaby about
|
| 17 |
+
death / killing / weapons / etc."
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
import re
|
| 21 |
+
|
| 22 |
+
# Terms that should never be the SUBJECT of a child's lullaby. Focused on
|
| 23 |
+
# death / violence / weapons / horror / substances β not mild words. Matched
|
| 24 |
+
# as whole words (so "grave" matches but "gravel" does not, "gun" matches but
|
| 25 |
+
# "begun" does not).
|
| 26 |
+
UNSAFE_TERMS = {
|
| 27 |
+
# death / dying
|
| 28 |
+
"death", "dead", "die", "dies", "died", "dying", "kill", "kills",
|
| 29 |
+
"killed", "killing", "murder", "murdered", "suicide", "corpse", "grave",
|
| 30 |
+
"graveyard", "coffin", "tomb", "funeral", "dead body", "hang", "hanging",
|
| 31 |
+
"noose", "drown", "drowned", "drowning",
|
| 32 |
+
# weapons / violence
|
| 33 |
+
"gun", "guns", "rifle", "pistol", "knife", "knives", "blade", "sword",
|
| 34 |
+
"bomb", "bombs", "explosion", "shoot", "shooting", "stab", "stabbed",
|
| 35 |
+
"weapon", "weapons", "blood", "bloody", "gore", "war", "battle",
|
| 36 |
+
"violence", "violent", "fight", "attack", "torture", "abuse",
|
| 37 |
+
# horror / occult
|
| 38 |
+
"demon", "demons", "devil", "satan", "hell", "ghost", "ghosts",
|
| 39 |
+
"haunted", "zombie", "zombies", "skull", "skeleton", "evil", "curse",
|
| 40 |
+
"cursed", "possessed", "sacrifice",
|
| 41 |
+
# substances / adult
|
| 42 |
+
"drug", "drugs", "cocaine", "heroin", "meth", "alcohol", "drunk",
|
| 43 |
+
"beer", "vodka", "cigarette", "cigarettes", "smoking", "weed",
|
| 44 |
+
"sex", "sexual", "naked", "nude",
|
| 45 |
+
# self-harm
|
| 46 |
+
"self harm", "self-harm", "cutting", "starve", "starving",
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
# Multi-word phrases need a substring check; single words use word-boundary.
|
| 50 |
+
_MULTIWORD = {t for t in UNSAFE_TERMS if " " in t or "-" in t}
|
| 51 |
+
_SINGLE = {t for t in UNSAFE_TERMS if t not in _MULTIWORD}
|
| 52 |
+
_WORD_RE = re.compile(
|
| 53 |
+
r"\b(" + "|".join(re.escape(t) for t in sorted(_SINGLE, key=len, reverse=True)) + r")\b",
|
| 54 |
+
re.IGNORECASE,
|
| 55 |
+
)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def find_unsafe_terms(text):
|
| 59 |
+
"""Return the list of unsafe terms found in `text` (empty if clean)."""
|
| 60 |
+
if not text:
|
| 61 |
+
return []
|
| 62 |
+
low = text.lower()
|
| 63 |
+
hits = set()
|
| 64 |
+
for phrase in _MULTIWORD:
|
| 65 |
+
if phrase in low:
|
| 66 |
+
hits.add(phrase)
|
| 67 |
+
for m in _WORD_RE.findall(low):
|
| 68 |
+
hits.add(m.lower())
|
| 69 |
+
return sorted(hits)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def is_safe(text):
|
| 73 |
+
"""True if `text` contains no unsafe terms."""
|
| 74 |
+
return len(find_unsafe_terms(text)) == 0
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def screen_inputs(loves, fears):
|
| 78 |
+
"""
|
| 79 |
+
Screen the user's loves+fears for a child's lullaby.
|
| 80 |
+
|
| 81 |
+
Returns (ok, message, bad_terms):
|
| 82 |
+
ok β True if both fields are safe
|
| 83 |
+
message β a gentle, kid-app-appropriate rejection message (or "")
|
| 84 |
+
bad_terms β the offending terms found (for logging/debugging)
|
| 85 |
+
"""
|
| 86 |
+
bad = []
|
| 87 |
+
bad += find_unsafe_terms(loves or "")
|
| 88 |
+
bad += find_unsafe_terms(fears or "")
|
| 89 |
+
bad = sorted(set(bad))
|
| 90 |
+
if bad:
|
| 91 |
+
msg = ("Let's keep the lullaby to gentle, cozy things that help with "
|
| 92 |
+
"sleep β like animals, the moon, or a favorite toy. Please "
|
| 93 |
+
"pick something soothing and try again.")
|
| 94 |
+
return (False, msg, bad)
|
| 95 |
+
return (True, "", [])
|
utils/trace.py
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
trace.py β opt-in pipeline tracer.
|
| 3 |
+
|
| 4 |
+
Records ONE complete Lolaby generation (inputs, every stage, final outputs)
|
| 5 |
+
into a timestamped folder so you can publish an example trace as an HF
|
| 6 |
+
Dataset for the "Sharing is Caring" hackathon badge.
|
| 7 |
+
|
| 8 |
+
How to use:
|
| 9 |
+
|
| 10 |
+
LULLABY_TRACE=1 python app.py
|
| 11 |
+
|
| 12 |
+
Generate a lullaby in the UI as you normally would. When the generation
|
| 13 |
+
finishes, a folder is written under ./traces/<timestamp>/ containing:
|
| 14 |
+
|
| 15 |
+
trace.json β the structured pipeline log
|
| 16 |
+
input_drawing.png β the drawing the user gave (if any)
|
| 17 |
+
output_lullaby.wav β the final audio
|
| 18 |
+
README.md β short note pointing at trace.json
|
| 19 |
+
|
| 20 |
+
That folder is what you upload to a Hugging Face Dataset.
|
| 21 |
+
|
| 22 |
+
Privacy posture:
|
| 23 |
+
- DISABLED by default (do nothing unless LULLABY_TRACE is set).
|
| 24 |
+
- Even when enabled, set LULLABY_TRACE_NO_IMAGES=1 to skip saving the
|
| 25 |
+
input drawing (records the trace structure but not the actual image).
|
| 26 |
+
|
| 27 |
+
Design note:
|
| 28 |
+
This is deliberately a module-level buffer accumulated by side-effects
|
| 29 |
+
rather than a context manager threaded through every call site. The
|
| 30 |
+
goal is MINIMAL changes to app.py β instrumentation calls are one
|
| 31 |
+
line each at the places that matter.
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
import json
|
| 35 |
+
import os
|
| 36 |
+
import time
|
| 37 |
+
from datetime import datetime
|
| 38 |
+
from pathlib import Path
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
ENABLED = os.environ.get("LULLABY_TRACE", "").lower() in ("1", "true", "yes")
|
| 42 |
+
SAVE_IMAGES = os.environ.get("LULLABY_TRACE_NO_IMAGES", "").lower() not in ("1", "true", "yes")
|
| 43 |
+
TRACE_ROOT = Path(os.environ.get("LULLABY_TRACE_DIR", "./traces"))
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
# Current run's accumulated state. Reset at the start of each generation
|
| 47 |
+
# via `begin()`. Read by `finalize()` at the end.
|
| 48 |
+
_buf = {}
|
| 49 |
+
_t0 = None
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def is_enabled():
|
| 53 |
+
return ENABLED
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def begin():
|
| 57 |
+
"""Reset the buffer at the start of a new generation. Cheap no-op if
|
| 58 |
+
tracing is disabled, so it's safe to call unconditionally."""
|
| 59 |
+
global _buf, _t0
|
| 60 |
+
if not ENABLED:
|
| 61 |
+
return
|
| 62 |
+
_t0 = time.monotonic()
|
| 63 |
+
_buf = {
|
| 64 |
+
"timestamp": datetime.utcnow().isoformat() + "Z",
|
| 65 |
+
"stages": {},
|
| 66 |
+
"stage_timings_ms": {},
|
| 67 |
+
}
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def stage(name, **fields):
|
| 71 |
+
"""Record a stage's metadata. Each call merges into stages[name]."""
|
| 72 |
+
if not ENABLED:
|
| 73 |
+
return
|
| 74 |
+
if name not in _buf["stages"]:
|
| 75 |
+
_buf["stages"][name] = {}
|
| 76 |
+
_buf["stages"][name].update(fields)
|
| 77 |
+
# Mark when this stage was last updated, relative to begin().
|
| 78 |
+
if _t0 is not None:
|
| 79 |
+
_buf["stage_timings_ms"][name] = round((time.monotonic() - _t0) * 1000)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def set_inputs(**kw):
|
| 83 |
+
"""Record the user-facing inputs (name, age, loves, fears, etc.).
|
| 84 |
+
Image arrays are NOT stored here β `save_input_drawing` handles them."""
|
| 85 |
+
if not ENABLED:
|
| 86 |
+
return
|
| 87 |
+
_buf["inputs"] = {k: v for k, v in kw.items() if v not in (None, "")}
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def finalize(output_audio=None, sample_rate=44100, error=None):
|
| 91 |
+
"""Write the trace folder. Called at the end of a generation, whether
|
| 92 |
+
or not it succeeded. Returns the trace folder path (or None if
|
| 93 |
+
tracing is disabled)."""
|
| 94 |
+
if not ENABLED:
|
| 95 |
+
return None
|
| 96 |
+
if _t0 is not None:
|
| 97 |
+
_buf["total_wall_time_ms"] = round((time.monotonic() - _t0) * 1000)
|
| 98 |
+
if error is not None:
|
| 99 |
+
_buf["error"] = str(error)
|
| 100 |
+
|
| 101 |
+
TRACE_ROOT.mkdir(parents=True, exist_ok=True)
|
| 102 |
+
stamp = datetime.utcnow().strftime("%Y-%m-%d_%H-%M-%S")
|
| 103 |
+
folder = TRACE_ROOT / stamp
|
| 104 |
+
folder.mkdir(exist_ok=True)
|
| 105 |
+
|
| 106 |
+
# Write trace.json
|
| 107 |
+
with open(folder / "trace.json", "w") as f:
|
| 108 |
+
json.dump(_buf, f, indent=2, default=_json_safe)
|
| 109 |
+
|
| 110 |
+
# Write output audio if provided
|
| 111 |
+
if output_audio is not None:
|
| 112 |
+
try:
|
| 113 |
+
import soundfile as sf
|
| 114 |
+
import numpy as np
|
| 115 |
+
audio = output_audio
|
| 116 |
+
if isinstance(audio, np.ndarray):
|
| 117 |
+
sf.write(folder / "output_lullaby.wav", audio, sample_rate)
|
| 118 |
+
except Exception as e:
|
| 119 |
+
print(f"[trace] could not write audio: {e}")
|
| 120 |
+
|
| 121 |
+
# Write a small README pointing at the trace file
|
| 122 |
+
readme = folder / "README.md"
|
| 123 |
+
readme.write_text(_README_TEMPLATE.format(stamp=stamp))
|
| 124 |
+
|
| 125 |
+
print(f"[trace] wrote {folder}/")
|
| 126 |
+
return folder
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def save_input_drawing(image_array):
|
| 130 |
+
"""Save the input drawing as input_drawing.png in the *current* run's
|
| 131 |
+
folder. Called inside the pipeline as soon as we know the image. Has
|
| 132 |
+
to be deferred-write because we don't know the folder name until
|
| 133 |
+
finalize() β so we stash the array here and write on finalize."""
|
| 134 |
+
if not ENABLED or not SAVE_IMAGES:
|
| 135 |
+
return
|
| 136 |
+
if image_array is None:
|
| 137 |
+
return
|
| 138 |
+
_buf["_pending_image"] = image_array
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def _flush_image(folder):
|
| 142 |
+
"""Internal: write the stashed image to the trace folder."""
|
| 143 |
+
arr = _buf.get("_pending_image")
|
| 144 |
+
if arr is None:
|
| 145 |
+
return
|
| 146 |
+
try:
|
| 147 |
+
from PIL import Image
|
| 148 |
+
import numpy as np
|
| 149 |
+
if isinstance(arr, np.ndarray):
|
| 150 |
+
if arr.ndim == 2:
|
| 151 |
+
img = Image.fromarray(arr).convert("RGB")
|
| 152 |
+
else:
|
| 153 |
+
img = Image.fromarray(arr[..., :3].astype(np.uint8))
|
| 154 |
+
img.save(folder / "input_drawing.png")
|
| 155 |
+
elif isinstance(arr, Image.Image):
|
| 156 |
+
arr.save(folder / "input_drawing.png")
|
| 157 |
+
except Exception as e:
|
| 158 |
+
print(f"[trace] could not write input drawing: {e}")
|
| 159 |
+
# Strip the binary out of the JSON-bound buffer.
|
| 160 |
+
del _buf["_pending_image"]
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def _json_safe(o):
|
| 164 |
+
"""Last-resort encoder for objects that aren't JSON-serializable."""
|
| 165 |
+
try:
|
| 166 |
+
import numpy as np
|
| 167 |
+
if isinstance(o, np.ndarray):
|
| 168 |
+
return f"<ndarray shape={o.shape} dtype={o.dtype}>"
|
| 169 |
+
if isinstance(o, (np.integer,)):
|
| 170 |
+
return int(o)
|
| 171 |
+
if isinstance(o, (np.floating,)):
|
| 172 |
+
return float(o)
|
| 173 |
+
except ImportError:
|
| 174 |
+
pass
|
| 175 |
+
return str(o)
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
_README_TEMPLATE = """# Lolaby β example generation trace
|
| 179 |
+
|
| 180 |
+
Captured: {stamp} (UTC)
|
| 181 |
+
|
| 182 |
+
## Files
|
| 183 |
+
|
| 184 |
+
- `trace.json` β full pipeline log: inputs, every stage's prompts and
|
| 185 |
+
outputs, stage timings, and the parsed lyric structure.
|
| 186 |
+
- `input_drawing.png` β the drawing the user gave Lola at the start of
|
| 187 |
+
this generation (omitted if no drawing was provided, or if traces
|
| 188 |
+
were captured with `LULLABY_TRACE_NO_IMAGES=1`).
|
| 189 |
+
- `output_lullaby.wav` β the audio Lola produced. WAV, 44.1 kHz mono.
|
| 190 |
+
|
| 191 |
+
## Note on reproducibility
|
| 192 |
+
|
| 193 |
+
This trace records one real generation end-to-end. The deterministic
|
| 194 |
+
stages (vision-model output with `do_sample=False`, DSP synth rendering
|
| 195 |
+
from parsed lyrics) will reproduce given the same inputs. The sampling
|
| 196 |
+
stages (the lyric model at `temperature=0.85`, Kokoro TTS) will produce
|
| 197 |
+
*similar but not identical* output on re-run β they're inherently
|
| 198 |
+
stochastic. The trace is a witness of one run, not a recipe.
|
| 199 |
+
|
| 200 |
+
## Pipeline at a glance
|
| 201 |
+
|
| 202 |
+
```
|
| 203 |
+
drawing ββΆ MiniCPM-V 4.6 ββΆ "what Lola saw"
|
| 204 |
+
β
|
| 205 |
+
+ name, fears, mood, instruments
|
| 206 |
+
βΌ
|
| 207 |
+
Fine-tuned Llama 3.2 3B (GGUF, llama.cpp)
|
| 208 |
+
β
|
| 209 |
+
parsed lyric + chords + tempo
|
| 210 |
+
β
|
| 211 |
+
βββββββββββββββββββ΄ββββββββββββββββββ
|
| 212 |
+
βΌ βΌ
|
| 213 |
+
Kokoro TTS DSP synths (custom)
|
| 214 |
+
βββββββββββββββββββ¬ββββββββββββββββββ
|
| 215 |
+
βΌ
|
| 216 |
+
output_lullaby.wav
|
| 217 |
+
```
|
| 218 |
+
"""
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
# Patch finalize() to call _flush_image at the right point. We do it this
|
| 222 |
+
# way to keep the begin/stage/finalize public API tiny.
|
| 223 |
+
_orig_finalize = finalize
|
| 224 |
+
def finalize(output_audio=None, sample_rate=44100, error=None): # noqa: F811
|
| 225 |
+
if not ENABLED:
|
| 226 |
+
return None
|
| 227 |
+
if _t0 is not None:
|
| 228 |
+
_buf["total_wall_time_ms"] = round((time.monotonic() - _t0) * 1000)
|
| 229 |
+
if error is not None:
|
| 230 |
+
_buf["error"] = str(error)
|
| 231 |
+
|
| 232 |
+
TRACE_ROOT.mkdir(parents=True, exist_ok=True)
|
| 233 |
+
stamp = datetime.utcnow().strftime("%Y-%m-%d_%H-%M-%S")
|
| 234 |
+
folder = TRACE_ROOT / stamp
|
| 235 |
+
folder.mkdir(exist_ok=True)
|
| 236 |
+
|
| 237 |
+
# Pull the stashed image out before we serialize the rest to JSON.
|
| 238 |
+
_flush_image(folder)
|
| 239 |
+
|
| 240 |
+
with open(folder / "trace.json", "w") as f:
|
| 241 |
+
json.dump(_buf, f, indent=2, default=_json_safe)
|
| 242 |
+
|
| 243 |
+
if output_audio is not None:
|
| 244 |
+
try:
|
| 245 |
+
import soundfile as sf
|
| 246 |
+
import numpy as np
|
| 247 |
+
audio = output_audio
|
| 248 |
+
if isinstance(audio, np.ndarray):
|
| 249 |
+
sf.write(folder / "output_lullaby.wav", audio, sample_rate)
|
| 250 |
+
except Exception as e:
|
| 251 |
+
print(f"[trace] could not write audio: {e}")
|
| 252 |
+
|
| 253 |
+
(folder / "README.md").write_text(_README_TEMPLATE.format(stamp=stamp))
|
| 254 |
+
print(f"[trace] wrote {folder}/")
|
| 255 |
+
return folder
|