Use embedding converter (#724)
* changes (#723) * Upload models to official assets repo --------- Co-authored-by: Harisreedhar <46858047+harisreedhar@users.noreply.github.com>
This commit is contained in:
Henry Ruhs
GitHub
parent
9f2b2926e3
commit
96282f192f
@ -22,7 +22,7 @@ from facefusion.processors.pixel_boost import explode_pixel_boost, implode_pixel
|
||||
from facefusion.processors.typing import FaceSwapperInputs
|
||||
from facefusion.program_helper import find_argument_group, suggest_face_swapper_pixel_boost_choices
|
||||
from facefusion.thread_helper import conditional_thread_semaphore
|
||||
from facefusion.typing import ApplyStateItem, Args, Embedding, Face, FaceLandmark5, InferencePool, ModelOptions, ModelSet, ProcessMode, QueuePayload, UpdateProgress, VisionFrame
|
||||
from facefusion.typing import ApplyStateItem, Args, Embedding, Face, InferencePool, ModelOptions, ModelSet, ProcessMode, QueuePayload, UpdateProgress, VisionFrame
|
||||
from facefusion.vision import read_image, read_static_image, read_static_images, unpack_resolution, write_image
|
||||
|
||||
MODEL_SET : ModelSet =\
|
||||
@ -35,11 +35,6 @@ MODEL_SET : ModelSet =\
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/blendswap_256.hash',
|
||||
'path': resolve_relative_path('../.assets/models/blendswap_256.hash')
|
||||
},
|
||||
'face_recognizer':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_w600k_r50.hash',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_w600k_r50.hash')
|
||||
}
|
||||
},
|
||||
'sources':
|
||||
@ -48,11 +43,6 @@ MODEL_SET : ModelSet =\
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/blendswap_256.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/blendswap_256.onnx')
|
||||
},
|
||||
'face_recognizer':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_w600k_r50.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_w600k_r50.onnx')
|
||||
}
|
||||
},
|
||||
'type': 'blendswap',
|
||||
@ -70,10 +60,10 @@ MODEL_SET : ModelSet =\
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/ghost_256_unet_1.hash',
|
||||
'path': resolve_relative_path('../.assets/models/ghost_256_unet_1.hash')
|
||||
},
|
||||
'face_recognizer':
|
||||
'embedding_converter':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_ghost.hash',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_ghost.hash')
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_converter_w600k_r50_ghost.hash',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_converter_w600k_r50_ghost.hash')
|
||||
}
|
||||
},
|
||||
'sources':
|
||||
@ -83,10 +73,10 @@ MODEL_SET : ModelSet =\
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/ghost_256_unet_1.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/ghost_256_unet_1.onnx')
|
||||
},
|
||||
'face_recognizer':
|
||||
'embedding_converter':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_ghost.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_ghost.onnx')
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_converter_w600k_r50_ghost.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_converter_w600k_r50_ghost.onnx')
|
||||
}
|
||||
},
|
||||
'type': 'ghost',
|
||||
@ -104,10 +94,10 @@ MODEL_SET : ModelSet =\
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/ghost_256_unet_2.hash',
|
||||
'path': resolve_relative_path('../.assets/models/ghost_256_unet_2.hash')
|
||||
},
|
||||
'face_recognizer':
|
||||
'embedding_converter':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_ghost.hash',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_ghost.hash')
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_converter_w600k_r50_ghost.hash',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_converter_w600k_r50_ghost.hash')
|
||||
}
|
||||
},
|
||||
'sources':
|
||||
@ -117,10 +107,10 @@ MODEL_SET : ModelSet =\
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/ghost_256_unet_2.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/ghost_256_unet_2.onnx')
|
||||
},
|
||||
'face_recognizer':
|
||||
'embedding_converter':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_ghost.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_ghost.onnx')
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_converter_w600k_r50_ghost.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_converter_w600k_r50_ghost.onnx')
|
||||
}
|
||||
},
|
||||
'type': 'ghost',
|
||||
@ -138,10 +128,10 @@ MODEL_SET : ModelSet =\
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/ghost_256_unet_3.hash',
|
||||
'path': resolve_relative_path('../.assets/models/ghost_256_unet_3.hash')
|
||||
},
|
||||
'face_recognizer':
|
||||
'embedding_converter':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_ghost.hash',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_ghost.hash')
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_converter_w600k_r50_ghost.hash',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_converter_w600k_r50_ghost.hash')
|
||||
}
|
||||
},
|
||||
'sources':
|
||||
@ -151,10 +141,10 @@ MODEL_SET : ModelSet =\
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/ghost_256_unet_3.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/ghost_256_unet_3.onnx')
|
||||
},
|
||||
'face_recognizer':
|
||||
'embedding_converter':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_ghost.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_ghost.onnx')
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_converter_w600k_r50_ghost.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_converter_w600k_r50_ghost.onnx')
|
||||
}
|
||||
},
|
||||
'type': 'ghost',
|
||||
@ -171,11 +161,6 @@ MODEL_SET : ModelSet =\
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/inswapper_128.hash',
|
||||
'path': resolve_relative_path('../.assets/models/inswapper_128.hash')
|
||||
},
|
||||
'face_recognizer':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0//arcface_w600k_r50.hash',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_w600k_r50.hash')
|
||||
}
|
||||
},
|
||||
'sources':
|
||||
@ -184,11 +169,6 @@ MODEL_SET : ModelSet =\
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/inswapper_128.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/inswapper_128.onnx')
|
||||
},
|
||||
'face_recognizer':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_w600k_r50.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_w600k_r50.onnx')
|
||||
}
|
||||
},
|
||||
'type': 'inswapper',
|
||||
@ -205,11 +185,6 @@ MODEL_SET : ModelSet =\
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/inswapper_128_fp16.hash',
|
||||
'path': resolve_relative_path('../.assets/models/inswapper_128_fp16.hash')
|
||||
},
|
||||
'face_recognizer':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_w600k_r50.hash',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_w600k_r50.hash')
|
||||
}
|
||||
},
|
||||
'sources':
|
||||
@ -218,11 +193,6 @@ MODEL_SET : ModelSet =\
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/inswapper_128_fp16.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/inswapper_128_fp16.onnx')
|
||||
},
|
||||
'face_recognizer':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_w600k_r50.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_w600k_r50.onnx')
|
||||
}
|
||||
},
|
||||
'type': 'inswapper',
|
||||
@ -240,10 +210,10 @@ MODEL_SET : ModelSet =\
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/simswap_256.hash',
|
||||
'path': resolve_relative_path('../.assets/models/simswap_256.hash')
|
||||
},
|
||||
'face_recognizer':
|
||||
'embedding_converter':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_simswap.hash',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_simswap.hash')
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_converter_w600k_r50_simswap.hash',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_converter_w600k_r50_simswap.hash')
|
||||
}
|
||||
},
|
||||
'sources':
|
||||
@ -253,10 +223,10 @@ MODEL_SET : ModelSet =\
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/simswap_256.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/simswap_256.onnx')
|
||||
},
|
||||
'face_recognizer':
|
||||
'embedding_converter':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_simswap.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_simswap.onnx')
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_converter_w600k_r50_simswap.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_converter_w600k_r50_simswap.onnx')
|
||||
}
|
||||
},
|
||||
'type': 'simswap',
|
||||
@ -274,10 +244,10 @@ MODEL_SET : ModelSet =\
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/simswap_512_unofficial.hash',
|
||||
'path': resolve_relative_path('../.assets/models/simswap_512_unofficial.hash')
|
||||
},
|
||||
'face_recognizer':
|
||||
'embedding_converter':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_simswap.hash',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_simswap.hash')
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_converter_w600k_r50_simswap.hash',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_converter_w600k_r50_simswap.hash')
|
||||
}
|
||||
},
|
||||
'sources':
|
||||
@ -287,10 +257,10 @@ MODEL_SET : ModelSet =\
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/simswap_512_unofficial.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/simswap_512_unofficial.onnx')
|
||||
},
|
||||
'face_recognizer':
|
||||
'embedding_converter':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_simswap.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_simswap.onnx')
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_converter_w600k_r50_simswap.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_converter_w600k_r50_simswap.onnx')
|
||||
}
|
||||
},
|
||||
'type': 'simswap',
|
||||
@ -307,11 +277,6 @@ MODEL_SET : ModelSet =\
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/uniface_256.hash',
|
||||
'path': resolve_relative_path('../.assets/models/uniface_256.hash')
|
||||
},
|
||||
'face_recognizer':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_w600k_r50.hash',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_w600k_r50.hash')
|
||||
}
|
||||
},
|
||||
'sources':
|
||||
@ -320,11 +285,6 @@ MODEL_SET : ModelSet =\
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/uniface_256.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/uniface_256.onnx')
|
||||
},
|
||||
'face_recognizer':
|
||||
{
|
||||
'url': 'https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/arcface_w600k_r50.onnx',
|
||||
'path': resolve_relative_path('../.assets/models/arcface_w600k_r50.onnx')
|
||||
}
|
||||
},
|
||||
'type': 'uniface',
|
||||
@ -465,13 +425,13 @@ def forward_swap_face(source_face : Face, crop_vision_frame : VisionFrame) -> Vi
|
||||
return crop_vision_frame
|
||||
|
||||
|
||||
def forward_calc_embedding(crop_vision_frame : VisionFrame) -> Embedding:
|
||||
face_recognizer = get_inference_pool().get('face_recognizer')
|
||||
def forward_convert_embedding(embedding : Embedding) -> Embedding:
|
||||
embedding_converter = get_inference_pool().get('embedding_converter')
|
||||
|
||||
with conditional_thread_semaphore():
|
||||
embedding = face_recognizer.run(None,
|
||||
embedding = embedding_converter.run(None,
|
||||
{
|
||||
'input': crop_vision_frame
|
||||
'input': embedding
|
||||
})[0]
|
||||
|
||||
return embedding
|
||||
@ -495,8 +455,7 @@ def prepare_source_embedding(source_face : Face) -> Embedding:
|
||||
model_type = get_model_options().get('type')
|
||||
|
||||
if model_type == 'ghost':
|
||||
source_vision_frame = read_static_image(get_first(state_manager.get_item('source_paths')))
|
||||
source_embedding, _ = calc_embedding(source_vision_frame, source_face.landmark_set.get('5/68'))
|
||||
source_embedding, _ = convert_embedding(source_face)
|
||||
source_embedding = source_embedding.reshape(1, -1)
|
||||
elif model_type == 'inswapper':
|
||||
model_path = get_model_options().get('sources').get('face_swapper').get('path')
|
||||
@ -504,18 +463,14 @@ def prepare_source_embedding(source_face : Face) -> Embedding:
|
||||
source_embedding = source_face.embedding.reshape((1, -1))
|
||||
source_embedding = numpy.dot(source_embedding, model_initializer) / numpy.linalg.norm(source_embedding)
|
||||
else:
|
||||
source_vision_frame = read_static_image(get_first(state_manager.get_item('source_paths')))
|
||||
_, source_normed_embedding = calc_embedding(source_vision_frame, source_face.landmark_set.get('5/68'))
|
||||
_, source_normed_embedding = convert_embedding(source_face)
|
||||
source_embedding = source_normed_embedding.reshape(1, -1)
|
||||
return source_embedding
|
||||
|
||||
|
||||
def calc_embedding(temp_vision_frame : VisionFrame, face_landmark_5 : FaceLandmark5) -> Tuple[Embedding, Embedding]:
|
||||
crop_vision_frame, matrix = warp_face_by_face_landmark_5(temp_vision_frame, face_landmark_5, 'arcface_112_v2', (112, 112))
|
||||
crop_vision_frame = crop_vision_frame / 127.5 - 1
|
||||
crop_vision_frame = crop_vision_frame[:, :, ::-1].transpose(2, 0, 1).astype(numpy.float32)
|
||||
crop_vision_frame = numpy.expand_dims(crop_vision_frame, axis = 0)
|
||||
embedding = forward_calc_embedding(crop_vision_frame)
|
||||
def convert_embedding(source_face : Face) -> Tuple[Embedding, Embedding]:
|
||||
embedding = source_face.embedding.reshape(-1, 512)
|
||||
embedding = forward_convert_embedding(embedding)
|
||||
embedding = embedding.ravel()
|
||||
normed_embedding = embedding / numpy.linalg.norm(embedding)
|
||||
return embedding, normed_embedding
|
||||
|
||||
Loading…
Reference in New Issue
Block a user