!pip install huggingface_hub~=0.23.3
!pip install transformers~=4.41.2

# Login to the HF Hub. We recommend using this login method
# to avoid the need for explicitly storing your HF token in variables
import huggingface_hub

huggingface_hub.interpreter_login()

# list all your inference endpoints
huggingface_hub.list_inference_endpoints()

# get an existing endpoint and check it's status
endpoint = huggingface_hub.get_inference_endpoint(
    name="gemma-1-1-2b-it-yci",  # the name of the endpoint
    namespace="MoritzLaurer",  # your user name or organization name
)
print(endpoint)

# Pause endpoint to stop billing
endpoint.pause()

# Resume and wait until the endpoint is ready
# endpoint.resume()
# endpoint.wait()

# Update the endpoint to a different GPU
# You can find the correct arguments for different hardware types in this table: https://huggingface.co/docs/inference-endpoints/pricing#gpu-instances
# endpoint.update(
#    instance_size="x1",
#    instance_type="nvidia-a100",  # nvidia-a10g
# )

from huggingface_hub import create_inference_endpoint


model_id = "google/gemma-1.1-2b-it"
endpoint_name = "gemma-1-1-2b-it-001"  # Valid Endpoint names must only contain lower-case characters, numbers or hyphens ("-") and are between 4 to 32 characters long.
namespace = "MoritzLaurer"  # your user or organization name


# check if endpoint with this name already exists from previous tests
available_endpoints_names = [endpoint.name for endpoint in huggingface_hub.list_inference_endpoints()]
if endpoint_name in available_endpoints_names:
    endpoint_exists = True
else:
    endpoint_exists = False
print("Does the endpoint already exist?", endpoint_exists)


# create new endpoint
if not endpoint_exists:
    endpoint = create_inference_endpoint(
        endpoint_name,
        repository=model_id,
        namespace=namespace,
        framework="pytorch",
        task="text-generation",
        # see the available hardware options here: https://huggingface.co/docs/inference-endpoints/pricing#pricing
        accelerator="gpu",
        vendor="aws",
        region="us-east-1",
        instance_size="x1",
        instance_type="nvidia-a10g",
        min_replica=0,
        max_replica=1,
        type="protected",
        # since the LLM is compatible with TGI, we specify that we want to use the latest TGI image
        custom_image={
            "health_route": "/health",
            "env": {"MODEL_ID": "/repository"},
            "url": "ghcr.io/huggingface/text-generation-inference:latest",
        },
    )
    print("Waiting for endpoint to be created")
    endpoint.wait()
    print("Endpoint ready")

# if endpoint with this name already exists, get and resume existing endpoint
else:
    endpoint = huggingface_hub.get_inference_endpoint(name=endpoint_name, namespace=namespace)
    if endpoint.status in ["paused", "scaledToZero"]:
        print("Resuming endpoint")
        endpoint.resume()
    print("Waiting for endpoint to start")
    endpoint.wait()
    print("Endpoint ready")

# access the endpoint url for API calls
print(endpoint.url)

>>> import requests
>>> from transformers import AutoTokenizer

>>> # paste your endpoint URL here or reuse endpoint.url if you created the endpoint programmatically
>>> API_URL = endpoint.url  # or paste link like "https://dz07884a53qjqb98.us-east-1.aws.endpoints.huggingface.cloud"
>>> HEADERS = {"Authorization": f"Bearer {huggingface_hub.get_token()}"}


>>> # function for standard http requests
>>> def query(payload=None, api_url=None):
...     response = requests.post(api_url, headers=HEADERS, json=payload)
...     return response.json()


>>> # define conversation input in messages format
>>> # you can also provide multiple turns between user and assistant
>>> messages = [
...     {"role": "user", "content": "Please write a short poem about open source for me."},
...     # {"role": "assistant", "content": "I am not in the mood."},
...     # {"role": "user", "content": "Can you please do this for me?"},
... ]

>>> # apply the chat template for the respective model
>>> model_id = "google/gemma-1.1-2b-it"
>>> tokenizer = AutoTokenizer.from_pretrained(model_id)
>>> messages_with_template = tokenizer.apply_chat_template(messages, tokenize=False)
>>> print("Your text input looks like this, after the chat template has been applied:\n")
>>> print(messages_with_template)

>>> # send standard http request to endpoint
>>> output = query(
...     payload={
...         "inputs": messages_with_template,
...         "parameters": {"temperature": 0.2, "max_new_tokens": 100, "seed": 42, "return_full_text": False},
...     },
...     api_url=API_URL,
... )

>>> print("The output from your API/Endpoint call:\n")
>>> print(output)

>>> API_URL_CHAT = API_URL + "/v1/chat/completions"

>>> output = query(
...     payload={
...         "messages": messages,
...         "model": "tgi",
...         "parameters": {"temperature": 0.2, "max_tokens": 100, "seed": 42},
...     },
...     api_url=API_URL_CHAT,
... )

>>> print("The output from your API/Endpoint call with the OpenAI-compatible messages API route:\n")
>>> print(output)

from huggingface_hub import InferenceClient

client = InferenceClient()

output = client.chat_completion(
    messages,  # the chat template is applied automatically, if your endpoint uses a TGI container
    model=API_URL,
    temperature=0.2,
    max_tokens=100,
    seed=42,
)

print("The output from your API/Endpoint call with the InferenceClient:\n")
print(output)

# pause the endpoint to stop billing
# endpoint.pause()

>>> !pip install Pillow  # for image processing

>>> from huggingface_hub import create_inference_endpoint

>>> model_id = "stabilityai/stable-diffusion-xl-base-1.0"
>>> endpoint_name = "stable-diffusion-xl-base-1-0-001"  # Valid Endpoint names must only contain lower-case characters, numbers or hyphens ("-") and are between 4 to 32 characters long.
>>> namespace = "MoritzLaurer"  # your user or organization name
>>> task = "text-to-image"

>>> # check if endpoint with this name already exists from previous tests
>>> available_endpoints_names = [endpoint.name for endpoint in huggingface_hub.list_inference_endpoints()]
>>> if endpoint_name in available_endpoints_names:
...     endpoint_exists = True
>>> else:
...     endpoint_exists = False
>>> print("Does the endpoint already exist?", endpoint_exists)


>>> # create new endpoint
>>> if not endpoint_exists:
...     endpoint = create_inference_endpoint(
...         endpoint_name,
...         repository=model_id,
...         namespace=namespace,
...         framework="pytorch",
...         task=task,
...         # see the available hardware options here: https://huggingface.co/docs/inference-endpoints/pricing#pricing
...         accelerator="gpu",
...         vendor="aws",
...         region="us-east-1",
...         instance_size="x1",
...         instance_type="nvidia-a100",
...         min_replica=0,
...         max_replica=1,
...         type="protected",
...     )
...     print("Waiting for endpoint to be created")
...     endpoint.wait()
...     print("Endpoint ready")

>>> # if endpoint with this name already exists, get existing endpoint
>>> else:
...     endpoint = huggingface_hub.get_inference_endpoint(name=endpoint_name, namespace=namespace)
...     if endpoint.status in ["paused", "scaledToZero"]:
...         print("Resuming endpoint")
...         endpoint.resume()
...     print("Waiting for endpoint to start")
...     endpoint.wait()
...     print("Endpoint ready")

>>> prompt = "A whimsical illustration of a fashionably dressed llama proudly holding a worn, vintage cookbook, with a warm cup of tea and a few freshly baked treats scattered around, set against a cozy background of rustic wood and blooming flowers."

>>> image = client.text_to_image(
...     prompt=prompt,
...     model=endpoint.url,  # "stabilityai/stable-diffusion-xl-base-1.0",
...     guidance_scale=8,
... )

>>> print("PROMPT: ", prompt)
>>> display(image.resize((image.width // 2, image.height // 2)))

endpoint.pause()

import base64
from io import BytesIO


def pil_image_to_base64(image):
    buffered = BytesIO()
    image.save(buffered, format="JPEG")
    img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
    return img_str


image_b64 = pil_image_to_base64(image)

from transformers import AutoProcessor

# load the processor
model_id_vlm = "HuggingFaceM4/idefics2-8b-chatty"
processor = AutoProcessor.from_pretrained(model_id_vlm)

# define the user messages
messages = [
    {
        "role": "user",
        "content": [
            {
                "type": "image"
            },  # the image is placed here in the prompt. You can add multiple images throughout the conversation.
            {"type": "text", "text": "Write a short limerick about this image."},
        ],
    },
]

# apply the chat template to the messages
prompt = processor.apply_chat_template(messages, add_generation_prompt=True)

# the chat template places a special "<image>" token at the position where the image should go
# here we replace the "<image>" token with the base64 encoded image string in the prompt
# to be able to send the image via an API request
image_input = f"data:image/jpeg;base64,{image_b64}"
image_input = f"![]({image_input})"
prompt = prompt.replace("<image>", image_input)

>>> from huggingface_hub import create_inference_endpoint

>>> endpoint_name = "idefics2-8b-chatty-001"
>>> namespace = "MoritzLaurer"
>>> task = "text-generation"

>>> # check if endpoint with this name already exists from previous tests
>>> available_endpoints_names = [endpoint.name for endpoint in huggingface_hub.list_inference_endpoints()]
>>> if endpoint_name in available_endpoints_names:
...     endpoint_exists = True
>>> else:
...     endpoint_exists = False
>>> print("Does the endpoint already exist?", endpoint_exists)


>>> if endpoint_exists:
...     endpoint = huggingface_hub.get_inference_endpoint(name=endpoint_name, namespace=namespace)
...     if endpoint.status in ["paused", "scaledToZero"]:
...         print("Resuming endpoint")
...         endpoint.resume()
...     print("Waiting for endpoint to start")
...     endpoint.wait()
...     print("Endpoint ready")

>>> else:
...     endpoint = create_inference_endpoint(
...         endpoint_name,
...         repository=model_id_vlm,
...         namespace=namespace,
...         framework="pytorch",
...         task=task,
...         accelerator="gpu",
...         vendor="aws",
...         region="us-east-1",
...         type="protected",
...         instance_size="x1",
...         instance_type="nvidia-a100",
...         min_replica=0,
...         max_replica=1,
...         custom_image={
...             "health_route": "/health",
...             "env": {
...                 "MAX_BATCH_PREFILL_TOKENS": "2048",
...                 "MAX_INPUT_LENGTH": "1024",
...                 "MAX_TOTAL_TOKENS": "1536",
...                 "MODEL_ID": "/repository",
...             },
...             "url": "ghcr.io/huggingface/text-generation-inference:latest",
...         },
...     )

...     print("Waiting for endpoint to be created")
...     endpoint.wait()
...     print("Endpoint ready")

>>> output = client.text_generation(prompt, model=model_id_vlm, max_new_tokens=200, seed=42)

>>> print(output)

endpoint.pause()