Playing With the SAM-Audio Model on My M1 MacBook

ERROR: Could not find a version that satisfies the requirement decord (from perception-models) (from versions: none)
ERROR: No matching distribution found for decord

❯ brew install miniconda

# I use bash, and this sets it up
❯ conda init bash

# refresh bash
❯ source ~/.bash_profile

# Prevent conda from hijacking every terminal session.
# For day to day use I am happy with pyenv as my default python manager.
❯ conda config --set auto_activate_base false

❯ mkdir sam-audio-playground

# Create a fresh virtual environment
# -n name of the environment
# -y always answer 'yes' to all prompts
❯ conda create -n sam-audio python=3.12 -y
❯ conda activate sam-audio

# purge the cache while pip is still conda's
❯ pip cache purge

# these are the standard commands
❯ conda deactivate
❯ conda env remove -n sam-audio

# try and remove everything that was cached
❯ conda clean -a -y

# find the env, if still there and really kill it
❯ rm -rf /opt/homebrew/Caskroom/miniconda/base/envs/sam-audio

# for extra safety, also purge the global non-conda pip cache
❯ pip cache purge

# stop pip complaining about virtual env
❯ conda env config vars set PIP_REQUIRE_VIRTUALENV=false

# reactivate to make it sticky
❯ conda activate sam-audio

❯ cd ~/work/sam-audio-playground
❯ conda activate sam-audio
❯ conda install -c conda-forge ffmpeg=7.1 -y

❯ cd ~/work/sam-audio-playground
❯ conda activate sam-audio
❯ pip install --pre torch torchvision torchaudio \
  --force-reinstall \
  --index-url https://download.pytorch.org/whl/nightly/cpu
❯ pip install --pre torchcodec --index-url https://download.pytorch.org/whl/nightly/cpu

❯ cd ~/work/sam-audio-playground
❯ conda activate sam-audio
# Force-build PyAV from source
❯ pip install av --no-binary av

❯ cd ~/work/sam-audio-playground
❯ conda activate sam-audio
❯ pip install git+https://github.com/facebookresearch/ImageBind.git
❯ pip install git+https://github.com/facebookresearch/dacvae.git
❯ pip install git+https://github.com/lematt1991/CLAP.git

# start from the parent folder
❯ cd ~/work/sam-audio-playground
❯ conda activate sam-audio
❯ gh repo clone facebookresearch/perception_models

# create a branch, so that I can still pull bug fixes etc to main
❯ cd perception_models
❯ git checkout -b "changed-by-me"

❯ nvim ./core/probe.py

# line 35
-from xformers.ops import fmha
+try:
+    from xformers.ops import fmha
+except ImportError:
+    fmha = None

# line 488
-        elif func._overloadpacket == fmha.flash.FwOp.OPERATOR:
+        elif fmha is not None and func._overloadpacket == fmha.flash.FwOp.OPERATOR:

❯ nvim ./core/transformer.py

# line 13
-from xformers.ops import AttentionBias, fmha
+try:
+    from xformers.ops import AttentionBias, fmha
+except ImportError:
+    class AttentionBias: pass
+    fmha = None

❯ nvim ./requirements.txt

# line 1
-numpy==2.1.2
+numpy<2.0.0
omegaconf==2.3.0
msgspec==0.19.0
rouge-score==0.1.2
sacrebleu==2.5.1
-sentencepiece==0.2.0
tiktoken==0.9.0

# line 22
-decord==0.6.0
opencv-python==4.11.0.86
pycocoevalcap==1.2
scikit-learn==1.6.1
scipy==1.15.2
-sentencepiece==0.2.0
+sentencepiece

❯ pip install -e .

# test that it works
❯ python -c "import core.vision_encoder.pe as pe; ..."

# installation created an egg directory; make sure it's not committed
❯ echo 'perception_models.egg-info/' >> .gitignore

# commit all changes
❯ git commit -a -m "feat: remove xformer and decord"

# start from the parent folder
❯ cd ~/work/sam-audio-playground
❯ conda activate sam-audio
❯ gh repo clone facebookresearch/sam-audio

# create a branch, so that I can still pull bug fixes etc to main
❯ cd sam-audio
❯ git checkout -b "changed-by-me"

❯ nvim ./pyproject.toml

# line 24
-    "perception-models@git+https://github.com/facebookresearch/perception_models@unpin-deps"
+    "perception-models@file:///Users/fritz/work/sam-audio-playground/perception_models",

❯ pip install -e .

# test that it works
❯ python -c "import sam_audio; ..."

# start from the parent folder
❯ cd ~/work/sam-audio-playground
❯ conda activate sam-audio

# create project scaffolding from copier
❯ copier copy https://github.com/gotofritz/copier-python-template --trust --vcs-ref=HEAD audio-playground

❯ cd audio-playground
❯ nvim ./pyproject.toml

# line 1
[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"

# line 14
dependencies = [
  "pydantic>=2.10.6",
  "pydantic-settings>=2.6.1",
  "click>=8.1.7",
  "rich>=14",
+  "sam-audio@file:///Users/fritz/work/sam-audio-playground/sam-audio",
+  "torch",
+  "torchaudio",
]

import click
import torch
import torchaudio
from sam_audio import SAMAudio, SAMAudioProcessor

from audio_playground.app_context import AppContext


@click.command(name="test-run")
@click.pass_context
def test_run(
    ctx: click.Context,
) -> None:
    """
    Just a test run
    """
    try:
        app_context: AppContext = ctx.obj
        logger = app_context.logger
        logger.info("Starting...")

        # up to here it's my usual CLI set up.
        # Now starts the SAM-audio specific code
        device = (
            torch.accelerator.current_accelerator().type
            if torch.accelerator.is_available()
            else "cpu"
        )
        logger.info(f"Using {device} device")

        model = (
            SAMAudio.from_pretrained(
                "facebook/sam-audio-large",
                map_location=device,
            )
            .to(device)
            .eval()
        )
        processor = SAMAudioProcessor.from_pretrained(
            "facebook/sam-audio-large",
        )

        # Load and process
        audio_path = "/Users/fritz/work/sam-audio-playground/wav/"
        source_path = audio_path + "sources/"
        dest_path = audio_path + "processed/"

        bass_source = source_path + "slap-bass.wav"
        bass_description = "The bass track"
        bass_dest = dest_path + "slap-"

        voice_source = source_path + "voice-over-clapping.wav"
        voice_description = "A man talking"
        voice_dest = dest_path + "voice-"

        with torch.inference_mode():
            inputs = processor(
                audios=[bass_source, voice_source],
                descriptions=[bass_description, voice_description],
            ).to(device)

            result = model.separate(
                inputs,
                predict_spans=False,
                reranking_candidates=8,
            )

            sr = processor.audio_sampling_rate

            target = result.target[0].unsqueeze(0).cpu()
            torchaudio.save(bass_dest + "target.wav", target, sr)
            residual = result.residual[0].unsqueeze(0).cpu()
            torchaudio.save(bass_dest + "residual.wav", residual, sr)

            target = result.target[1].unsqueeze(0).cpu()
            torchaudio.save(voice_dest + "target.wav", target, sr)
            residual = result.residual[1].unsqueeze(0).cpu()
            torchaudio.save(voice_dest + "residual.wav", residual, sr)

        logger.info("Done")

    except Exception as e:
        click.echo(f"CLI Error: {str(e)}")
        ctx.exit(1)

# ensures it files fetched from Hugging Face are
# not downloaded at every run
❯ conda env config vars set HF_DATASETS_OFFLINE=1
❯ conda env config vars set TRANSFORMERS_OFFLINE=1
# reactivate to make it sticky
❯ conda activate sam-audio
❯ pip install -e .
❯ audio-playground test-run

❯ mkdir -p "$CONDA_PREFIX/etc/conda/activate.d"
❯ cat <<EOF > $CONDA_PREFIX/etc/conda/activate.d/env_vars.sh
#!/bin/sh

export HF_DATASETS_OFFLINE=1
export TRANSFORMERS_OFFLINE=1
export PIP_REQUIRE_VIRTUALENV=false
EOF