Setup conda on ubuntu

# Install conda
mkdir -p ~/miniconda3
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda3/miniconda.sh
bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3
rm ~/miniconda3/miniconda.sh
source ~/miniconda3/bin/activate
conda init --all
bash

Setup python environment for xinference

conda create -n xinference python=3.12 -y
conda activate xinference

Install xinference and vllm

pip install "xinference[all]"

Setup cluster

conda activate xinference

# list available models
xinference list