Audio2PhotoReal is an innovative deep learning project open-sourced by Facebook Research (Meta Research) that implements an end-to-end synthesis system for generating realistic human avatars from audio. The core capability of this project is to generate realistic full-body human animations, including facial expressions and body poses, based on voice input.
The project adopts a multi-model collaborative architecture design:
The project provides complete datasets for four characters:
Each character contains approximately 26-30 dialogue scenes, each scene containing:
*audio.wav: Stereo audio file (48kHz)
- Channel 0: Current character audio
- Channel 1: Dialogue partner audio
*body_pose.npy: (T × 104) Joint angle array
*face_expression.npy: (T × 256) Facial encoding array
*missing_face_frames.npy: Indices of missing/corrupted facial frames
data_stats.pth: Mean and standard deviation statistics for each modality
train_idx = list(range(0, len(data_dict["data"]) - 6))
val_idx = list(range(len(data_dict["data"]) - 6, len(data_dict["data"]) - 4))
test_idx = list(range(len(data_dict["data"]) - 4, len(data_dict["data"])))
# Create environment
conda create --name a2p_env python=3.9
conda activate a2p_env
# Install dependencies
sh demo/install.sh
# Run demo
python -m demo.demo
# Environment configuration
conda create --name a2p_env python=3.9
conda activate a2p_env
pip install -r scripts/requirements.txt
# Download necessary models
sh scripts/download_prereq.sh
# Install PyTorch3D
pip install "git+https://github.com/facebookresearch/pytorch3d.git"
# Download a single dataset
curl -L https://github.com/facebookresearch/audio2photoreal/releases/download/v1.0/<person_id>.zip -o <person_id>.zip
unzip <person_id>.zip -d dataset/
# Download all datasets
sh scripts/download_alldatasets.sh
# Download pre-trained models
sh scripts/download_allmodels.sh
python -m train.train_diffusion \
--save_dir checkpoints/diffusion/c1_face_test \
--data_root ./dataset/PXB184/ \
--batch_size 4 \
--dataset social \
--data_format face \
--layers 8 \
--heads 8 \
--timestep_respacing '' \
--max_seq_length 600
python -m train.train_diffusion \
--save_dir checkpoints/diffusion/c1_pose_test \
--data_root ./dataset/PXB184/ \
--lambda_vel 2.0 \
--batch_size 4 \
--dataset social \
--add_frame_cond 1 \
--data_format pose \
--layers 6 \
--heads 8 \
--timestep_respacing '' \
--max_seq_length 600
python -m train.train_vq \
--out_dir checkpoints/vq/c1_vq_test \
--data_root ./dataset/PXB184/ \
--lr 1e-3 \
--code_dim 1024 \
--output_emb_width 64 \
--depth 4 \
--dataname social \
--loss_vel 0.0 \
--data_format pose \
--batch_size 4 \
--add_frame_cond 1 \
--max_seq_length 600
python -m train.train_guide \
--out_dir checkpoints/guide/c1_trans_test \
--data_root ./dataset/PXB184/ \
--batch_size 4 \
--resume_pth checkpoints/vq/c1_vq_test/net_iter300000.pth \
--add_frame_cond 1 \
--layers 6 \
--lr 2e-4 \
--gn \
--dim 64
python -m sample.generate \
--model_path checkpoints/diffusion/c1_face/model000155000.pt \
--num_samples 10 \
--num_repetitions 5 \
--timestep_respacing ddim500 \
--guidance_param 10.0
python -m sample.generate \
--model_path checkpoints/diffusion/c1_pose/model000340000.pt \
--resume_trans checkpoints/guide/c1_pose/checkpoints/iter-0100000.pt \
--num_samples 10 \
--num_repetitions 5 \
--timestep_respacing ddim500 \
--guidance_param 2.0
python -m sample.generate \
--model_path checkpoints/diffusion/c1_pose/model000340000.pt \
--resume_trans checkpoints/guide/c1_pose/checkpoints/iter-0100000.pt \
--num_samples 10 \
--num_repetitions 5 \
--timestep_respacing ddim500 \
--guidance_param 2.0 \
--face_codes ./checkpoints/diffusion/c1_face/samples_c1_face_000155000_seed10_/results.npy \
--pose_codes ./checkpoints/diffusion/c1_pose/samples_c1_pose_000340000_seed10_guide_iter-0100000.pt/results.npy \
--plot
python -m visualize.render_anno \
--save_dir vis_anno_test \
--data_root dataset/PXB184 \
--max_seq_length 600