app.py

import streamlit as st
import subprocess
import sys

def install(package):
    subprocess.check_call([sys.executable, "-m", "pip", "install", package])
install("llama-cpp-python")

from llama_cpp import Llama
prompt = st.chat_input("Say something")
if prompt:
    llm = Llama(model_path="Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf")
    r=llm(prompt, max_tokens=1000)
    st.write(f"User has sent the following prompt: {prompt} with response: {r["choices"][0]["text"]} ")