Module `spin_sdk.wit.imports.llm`

A WASI interface dedicated to performing inferencing for Large Language Models.

Expand source code

"""
A WASI interface dedicated to performing inferencing for Large Language Models.
"""
from typing import TypeVar, Generic, Union, Optional, Union, Protocol, Tuple, List, Any, Self
from enum import Flag, Enum, auto
from dataclasses import dataclass
from abc import abstractmethod
import weakref

from ..types import Result, Ok, Err, Some


@dataclass
class InferencingParams:
    """
    Inference request parameters
    """
    max_tokens: int
    repeat_penalty: float
    repeat_penalty_last_n_token_count: int
    temperature: float
    top_k: int
    top_p: float


@dataclass
class ErrorModelNotSupported:
    pass


@dataclass
class ErrorRuntimeError:
    value: str


@dataclass
class ErrorInvalidInput:
    value: str


Error = Union[ErrorModelNotSupported, ErrorRuntimeError, ErrorInvalidInput]
"""
The set of errors which may be raised by functions in this interface
"""


@dataclass
class InferencingUsage:
    """
    Usage information related to the inferencing result
    """
    prompt_token_count: int
    generated_token_count: int

@dataclass
class InferencingResult:
    """
    An inferencing result
    """
    text: str
    usage: InferencingUsage

@dataclass
class EmbeddingsUsage:
    """
    Usage related to an embeddings generation request
    """
    prompt_token_count: int

@dataclass
class EmbeddingsResult:
    """
    Result of generating embeddings
    """
    embeddings: List[List[float]]
    usage: EmbeddingsUsage


def infer(model: str, prompt: str, params: Optional[InferencingParams]) -> InferencingResult:
    """
    Perform inferencing using the provided model and prompt with the given optional params
    
    Raises: `spin_sdk.wit.types.Err(spin_sdk.wit.imports.llm.Error)`
    """
    raise NotImplementedError

def generate_embeddings(model: str, text: List[str]) -> EmbeddingsResult:
    """
    Generate embeddings for the supplied list of text
    
    Raises: `spin_sdk.wit.types.Err(spin_sdk.wit.imports.llm.Error)`
    """
    raise NotImplementedError

Global variables

var Error: The set of errors which may be raised by functions in this interface

Functions

def generate_embeddings(model: str, text: List[str]) ‑> EmbeddingsResult

Generate embeddings for the supplied list of text

Raises: Err(Error)

Expand source code

def generate_embeddings(model: str, text: List[str]) -> EmbeddingsResult:
    """
    Generate embeddings for the supplied list of text
    
    Raises: `spin_sdk.wit.types.Err(spin_sdk.wit.imports.llm.Error)`
    """
    raise NotImplementedError

def infer(model: str, prompt: str, params: Optional[InferencingParams]) ‑> InferencingResult

Perform inferencing using the provided model and prompt with the given optional params

Raises: Err(Error)

Expand source code

def infer(model: str, prompt: str, params: Optional[InferencingParams]) -> InferencingResult:
    """
    Perform inferencing using the provided model and prompt with the given optional params
    
    Raises: `spin_sdk.wit.types.Err(spin_sdk.wit.imports.llm.Error)`
    """
    raise NotImplementedError

Classes

class EmbeddingsResult (embeddings: List[List[float]], usage: EmbeddingsUsage)

Result of generating embeddings

Expand source code

@dataclass
class EmbeddingsResult:
    """
    Result of generating embeddings
    """
    embeddings: List[List[float]]
    usage: EmbeddingsUsage

Class variables

var embeddings : List[List[float]]
var usage : EmbeddingsUsage

class EmbeddingsUsage (prompt_token_count: int)

Usage related to an embeddings generation request

Expand source code

@dataclass
class EmbeddingsUsage:
    """
    Usage related to an embeddings generation request
    """
    prompt_token_count: int

Class variables

var prompt_token_count : int

class ErrorInvalidInput (value: str)

ErrorInvalidInput(value: str)

Expand source code

@dataclass
class ErrorInvalidInput:
    value: str

Class variables

var value : str

class ErrorModelNotSupported

ErrorModelNotSupported()

Expand source code

@dataclass
class ErrorModelNotSupported:
    pass

class ErrorRuntimeError (value: str)

ErrorRuntimeError(value: str)

Expand source code

@dataclass
class ErrorRuntimeError:
    value: str

Class variables

var value : str

class InferencingParams (max_tokens: int, repeat_penalty: float, repeat_penalty_last_n_token_count: int, temperature: float, top_k: int, top_p: float)

Inference request parameters

Expand source code

@dataclass
class InferencingParams:
    """
    Inference request parameters
    """
    max_tokens: int
    repeat_penalty: float
    repeat_penalty_last_n_token_count: int
    temperature: float
    top_k: int
    top_p: float

Class variables

var max_tokens : int
var repeat_penalty : float
var repeat_penalty_last_n_token_count : int
var temperature : float
var top_k : int
var top_p : float

class InferencingResult (text: str, usage: InferencingUsage)

An inferencing result

Expand source code

@dataclass
class InferencingResult:
    """
    An inferencing result
    """
    text: str
    usage: InferencingUsage

Class variables

var text : str
var usage : InferencingUsage

class InferencingUsage (prompt_token_count: int, generated_token_count: int)

Usage information related to the inferencing result

Expand source code

@dataclass
class InferencingUsage:
    """
    Usage information related to the inferencing result
    """
    prompt_token_count: int
    generated_token_count: int

Class variables

var generated_token_count : int
var prompt_token_count : int