refactor(openai.go, tokens.go): streamline comments and function desc…

…riptions

- Simplify and clarify documentation comments across various functions.
- Remove redundant explanations and ensure comments are concise and directly related to function purposes.
- Adjust variable and function naming for better clarity and consistency in codebase.

services/openai/openai.go

@@ -15,29 +15,18 @@ import (
 )
 
 const (
-	// DefaultModel represents the pre-configured model identifier to be used by the
-	// Service when no specific model is provided. This default serves as a fallback
-	// and ensures that the Service has a valid starting point for its operations
-	// related to AI-powered text generation or completion tasks.
+	// DefaultModel specifies the default language model used for generating
+	// completions.
 	DefaultModel = openai.GPT3Dot5Turbo
 
-	// DefaultMaxTokens represents the initial setting for the maximum number of
-	// tokens that can be generated by the service. This value is used as a default
-	// when no specific maximum is set through service configuration options. It
-	// ensures that the output from token generation does not exceed a predefined
-	// length, providing a balance between performance and output detail.
+	// DefaultMaxTokens specifies the default maximum number of tokens for a
+	// request.
 	DefaultMaxTokens = 512
 )
 
-// Service orchestrates the generation of textual content using a specified
-// model from OpenAI. It manages the creation of requests, execution of the
-// underlying model, and interpretation of the results to produce coherent
-// output. Service provides options for customizing the model, client
-// configuration, logging, and token limits. It supports both standard and
-// chat-based completions, adapting to the input prompts provided by users
-// through a defined context. The Service ensures that the generated content
-// adheres to specified constraints such as token limits, while also handling
-// error scenarios and logging usage information.
+// Service provides functionalities for generating text completions using OpenAI
+// models, with options to customize the model, maximum tokens, and logging
+// behavior.
 type Service struct {
 	client    *openai.Client
 	model     string
@@ -46,65 +35,40 @@ type Service struct {
 	log       *slog.Logger
 }
 
-// Option represents a configuration setting that can be applied to customize
-// the behavior of a Service. It acts as a modifier for a Service instance,
-// allowing specific aspects such as the model used, the client configuration,
-// maximum token count, or logging details to be set according to the user's
-// preferences. Options are designed to be passed to the Service constructor,
-// enabling a flexible and fluent API for Service configuration.
+// Option modifies the configuration of a [Service].
 type Option func(*Service)
 
-// Model configures the model to be used by the Service for generating content.
-// It accepts a model identifier as a parameter and returns an Option that, when
-// applied, sets the model field in the Service struct to the specified
-// identifier.
+// Model sets the model to be used by the [Service].
 func Model(model string) Option {
 	return func(s *Service) {
 		s.model = model
 	}
 }
 
-// Client configures a Service with the provided OpenAI client instance. It is
-// used as an option when creating a new Service. This allows for the use of a
-// custom OpenAI client instead of the default one that would be created using
-// an API key.
+// Client sets the [*openai.Client] for the [Service].
 func Client(c *openai.Client) Option {
 	return func(s *Service) {
 		s.client = c
 	}
 }
 
-// MaxTokens sets the maximum number of tokens to use for generating content
-// with the service. It configures the service instance by applying a limit on
-// the token count for output generation, which can affect the verbosity and
-// detail of the generated text. The provided maximum should not exceed the
-// model's inherent token limit. If it does, the service will automatically
-// adjust to use the maximum allowed by the model or the remaining tokens after
-// accounting for input prompt tokens, whichever is lower. This option helps in
-// managing resource usage and controlling the length of generated content.
+// MaxTokens sets the maximum number of tokens that can be used in a request.
 func MaxTokens(max int) Option {
 	return func(s *Service) {
 		s.maxTokens = max
 	}
 }
 
-// WithLogger configures a logging handler for the service, allowing the service
-// to log its activities. It accepts a logging handler and returns an option
-// that can be passed to the service constructor.
+// WithLogger configures the service to use a specified logger. Returns an
+// [Option].
 func WithLogger(h slog.Handler) Option {
 	return func(s *Service) {
 		s.log = slog.New(h)
 	}
 }
 
-// New initializes a new instance of Service with the provided API key and
-// options, returning a pointer to the service and any error encountered during
-// the setup. It applies the given options to customize the Service, such as
-// setting a specific model, client, maximum tokens, or logger. If no client is
-// provided, it creates a new client using the API key. If no model is
-// specified, it defaults to the predefined model. The function also ensures
-// that an appropriate tokenizer and a non-nil logger are set up for the service
-// before returning.
+// New initializes and returns a new [*Service] configured with the provided API
+// key and options.
 func New(apiKey string, opts ...Option) (*Service, error) {
 	svc := Service{maxTokens: DefaultMaxTokens}
 	for _, opt := range opts {
@@ -133,12 +97,8 @@ func New(apiKey string, opts ...Option) (*Service, error) {
 	return &svc, nil
 }
 
-// GenerateDoc creates a document based on the context provided by the caller.
-// It logs the generation process, constructs a request tailored to the input
-// context, and invokes the appropriate model to generate content. The function
-// returns the generated text or an error if the generation process fails. The
-// operation respects a timeout and ensures that the size of the generated
-// content does not exceed predefined token limits.
+// GenerateDoc creates documentation text based on the provided context and
+// returns it as a string.
 func (svc *Service) GenerateDoc(ctx generate.Context) (string, error) {
 	svc.log.Debug(fmt.Sprintf("[OpenAI] Generating docs for %s (%s)", ctx.Input().Identifier, ctx.Input().Language))
 
@@ -296,10 +256,8 @@ func isChatModel(model string) bool {
 	return strings.HasPrefix(model, "gpt-")
 }
 
-// MaxTokensForModel retrieves the maximum number of tokens allowed for a given
-// model. If the specified model is not found in the predefined list, it returns
-// the default maximum token count. This limit is crucial for ensuring that
-// token generation stays within the bounds set by the model's capabilities.
+// MaxTokensForModel retrieves the maximum number of tokens allowed for a
+// specified model.
 func MaxTokensForModel(model string) int {
 	if t, ok := modelMaxTokens[model]; ok {
 		return t

services/openai/tokens.go

@@ -15,14 +15,8 @@ var (
 	codecs    = make(map[string]tokenizer.Codec)
 )
 
-// ChatTokens calculates the total number of tokens that would be generated by
-// encoding a sequence of chat completion messages using a specified model's
-// tokenizer. It takes into account any additional tokens required for the
-// message names and content, based on the model's specific tokenization rules.
-// The function returns the calculated token count alongside any error
-// encountered during the tokenization process. If an error occurs, it includes
-// additional context to clarify that the error happened while encoding a
-// message.
+// ChatTokens calculates the total number of tokens used by a list of
+// [openai.ChatCompletionMessage] for a given model.
 func ChatTokens(model string, messages []openai.ChatCompletionMessage) (int, error) {
 	codec, err := getCodec(model)
 	if err != nil {
@@ -73,10 +67,9 @@ func getCodec(model string) (tokenizer.Codec, error) {
 	return codec, nil
 }
 
-// PromptTokens calculates the number of tokens in a given prompt using the
-// specified model's tokenizer. It returns the token count and any error
-// encountered during the tokenization process. If an error occurs, the token
-// count is not guaranteed to be accurate.
+// PromptTokens calculates the number of tokens in a given prompt based on the
+// specified model and returns the token count along with any error encountered
+// during encoding.
 func PromptTokens(model string, prompt string) (int, error) {
 	codec, err := getCodec(model)
 	if err != nil {