示例

这个模型绝不是最先进的模型，但是仍然能够产生合理的图像描述结果。它主要是为了证明 ? FlaxVisionEncoderDecoder 框架的概念而进行的微调。

模型的使用方法如下：

import requests
from PIL import Image
from transformers import ViTFeatureExtractor, AutoTokenizer, FlaxVisionEncoderDecoderModel

loc = "ydshieh/vit-gpt2-coco-en"

feature_extractor = ViTFeatureExtractor.from_pretrained(loc)
tokenizer = AutoTokenizer.from_pretrained(loc)
model = FlaxVisionEncoderDecoderModel.from_pretrained(loc)

# We will verify our results on an image of cute cats
url = "http://images.cocodataset.org/val2017/000000039769.jpg"
with Image.open(requests.get(url, stream=True).raw) as img:
    pixel_values = feature_extractor(images=img, return_tensors="np").pixel_values

def generate_step(pixel_values):

    output_ids = model.generate(pixel_values, max_length=16, num_beams=4).sequences
    preds = tokenizer.batch_decode(output_ids, skip_special_tokens=True)
    preds = [pred.strip() for pred in preds]

    return preds

preds = generate_step(pixel_values)
print(preds)

# should produce
# ['a cat laying on top of a couch next to another cat']