Only input text to VLM?

[laolongboy]

X2Edit/src/pipeline_1024.py

Lines 199 to 234 in cf25bac

	def _get_qwenvl_prompt_embeds(
	self,
	prompt: Union[str, List[str]] = None,
	image: Image = None,
	num_images_per_prompt: int = 1,
	max_sequence_length: int = 512,
	device: Optional[torch.device] = None,
	dtype: Optional[torch.dtype] = None,
	):
	ref = False
	Instructions = {"Text input":"","Instruction editing description":prompt,"image input":"yes"}
	contents=[{"type": "text", "text": str(Instructions)}]
	if ref:
	new_image = image.resize((256, 256),)
	contents.append({"type": "image", "image": new_image})
	message = [{"role": "user", "content": contents}]
	device = device or self._execution_device
	dtype = dtype or self.text_encoder.dtype
	self.text_encoder_qwenvl = self.text_encoder_qwenvl.to(device=device, dtype=dtype)
	self.proj = self.proj.to(device=device, dtype=dtype)
	text = self.tokenizer_qwenvl.apply_chat_template(message, tokenize=False, add_generation_prompt=True)
	if ref:
	inputs = self.tokenizer_qwenvl(text=[text],
	images=[new_image],
	videos=None, padding="max_length", max_length=max_sequence_length, truncation=True, return_tensors="pt").to(device)
	else:
	inputs = self.tokenizer_qwenvl(text=[text],
	images=None,
	videos=None, padding="max_length", max_length=max_sequence_length, truncation=True, return_tensors="pt").to(device)
	output_hidden_state_all = self.text_encoder_qwenvl.generate(**inputs, max_new_tokens=1,output_hidden_states=True,return_dict_in_generate=True)
	text_embeddings = torch.stack(output_hidden_state_all["hidden_states"][0], dim=1)
	pooled_prompt_embeds, prompt_embeds = self.proj(text_embeddings)
	return pooled_prompt_embeds, prompt_embeds

Hardcodes ref = False, so the vlm doesn't get any info from reference image?

[CVPIE]

Yes, the reference image is optional input.