llm.py


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

import json
from collections.abc import Iterator

import requests

class LLM:
    def __init__(self, url: str) -> None:
        self.url = url

    def chat(self, messages: list[dict[str, str]], max_tokens: int = 256) -> Iterator[str]:
        response = requests.post(
            self.url,
            json = {
                "messages": messages,
                "max_tokens": max_tokens,
                "temperature": 0.7,
                "stream": True,
            },
            stream=True,
        )
        response.raise_for_status()

        for line in response.iter_lines():
            if not line:
                continue
            
            if not line.startswith(b"data: "):
                continue

            data = line[6:]

            if data == b"[DONE]":
                break

            chunk = json.loads(data)

            delta = chunk["choices"][0]["delta"]
            content = delta.get("content")

            if content:
                yield content