File size: 1,805 Bytes
192f447 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 |
import streamlit as st
from llama_cpp import Llama
repo_ir = "Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF"
llm = Llama.from_pretrained(
repo_id=repo_ir,
filename="qwen2.5-coder-1.5b-instruct-q8_0.gguf",
verbose=True,
use_mmap=True,
use_mlock=True,
n_threads=4,
n_threads_batch=4,
n_ctx=8000,
)
print(f"{repo_ir} loaded successfully. ✅")
# Streamed response emulator
def response_generator(messages):
completion = llm.create_chat_completion(
messages, max_tokens=2048, stream=True, temperature=0.7, top_p=0.95
)
for message in completion:
delta = message["choices"][0]["delta"]
if "content" in delta:
yield delta["content"]
st.title("CSV TO SQL")
# Initialize chat history
if "messages" not in st.session_state:
st.session_state.messages = []
# Display chat messages from history on app rerun
for message in st.session_state.messages:
with st.chat_message(message["role"]):
st.markdown(message["content"])
# Accept user input
if prompt := st.chat_input("What is up?"):
# Add user message to chat history
st.session_state.messages.append({"role": "user", "content": prompt})
# Display user message in chat message container
with st.chat_message("user"):
st.markdown(prompt)
messages = [{"role": "system", "content": "You are a helpful assistant"}]
for val in st.session_state.messages:
messages.append(val)
messages.append({"role": "user", "content": prompt})
# Display assistant response in chat message container
with st.chat_message("assistant"):
response = st.write_stream(response_generator(messages=messages))
# Add assistant response to chat history
st.session_state.messages.append({"role": "assistant", "content": response})
|