From acc2dc89e184d19278e97ef7c62867cb0cc63539 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Wed, 9 Oct 2024 15:53:53 -0400 Subject: [PATCH 1/9] feat: add datasets to sdkserver Signed-off-by: Grant Linville --- pkg/cli/gptscript.go | 2 + pkg/gptscript/gptscript.go | 7 + pkg/sdkserver/datasets.go | 329 +++++++++++++++++++++++++++++++++++++ pkg/sdkserver/routes.go | 6 + 4 files changed, 344 insertions(+) create mode 100644 pkg/sdkserver/datasets.go diff --git a/pkg/cli/gptscript.go b/pkg/cli/gptscript.go index 66719adc..19ab7b29 100644 --- a/pkg/cli/gptscript.go +++ b/pkg/cli/gptscript.go @@ -75,6 +75,7 @@ type GPTScript struct { SaveChatStateFile string `usage:"A file to save the chat state to so that a conversation can be resumed with --chat-state" local:"true"` DefaultModelProvider string `usage:"Default LLM model provider to use, this will override OpenAI settings"` GithubEnterpriseHostname string `usage:"The host name for a Github Enterprise instance to enable for remote loading" local:"true"` + DatasetToolRepo string `usage:"The repo to use for dataset tools" default:"github.com/gptscript-ai/datasets" local:"true"` readData []byte } @@ -146,6 +147,7 @@ func (r *GPTScript) NewGPTScriptOpts() (gptscript.Options, error) { Workspace: r.Workspace, DisablePromptServer: r.UI, DefaultModelProvider: r.DefaultModelProvider, + DatasetToolRepo: r.DatasetToolRepo, } if r.Confirm { diff --git a/pkg/gptscript/gptscript.go b/pkg/gptscript/gptscript.go index 679eb503..a6bff8f5 100644 --- a/pkg/gptscript/gptscript.go +++ b/pkg/gptscript/gptscript.go @@ -32,6 +32,8 @@ import ( var log = mvl.Package() +const defaultDatasetToolRepo = "github.com/gptscript-ai/datasets" + type GPTScript struct { Registry *llm.Registry Runner *runner.Runner @@ -51,6 +53,7 @@ type Options struct { CredentialContexts []string Quiet *bool Workspace string + DatasetToolRepo string DisablePromptServer bool Env []string } @@ -66,6 +69,7 @@ func Complete(opts ...Options) Options { result.CredentialContexts = opt.CredentialContexts result.Quiet = types.FirstSet(opt.Quiet, result.Quiet) result.Workspace = types.FirstSet(opt.Workspace, result.Workspace) + result.DatasetToolRepo = types.FirstSet(opt.DatasetToolRepo, result.DatasetToolRepo) result.Env = append(result.Env, opt.Env...) result.DisablePromptServer = types.FirstSet(opt.DisablePromptServer, result.DisablePromptServer) result.DefaultModelProvider = types.FirstSet(opt.DefaultModelProvider, result.DefaultModelProvider) @@ -80,6 +84,9 @@ func Complete(opts ...Options) Options { if len(result.CredentialContexts) == 0 { result.CredentialContexts = []string{credentials.DefaultCredentialContext} } + if result.DatasetToolRepo == "" { + result.DatasetToolRepo = defaultDatasetToolRepo + } return result } diff --git a/pkg/sdkserver/datasets.go b/pkg/sdkserver/datasets.go new file mode 100644 index 00000000..2f40971a --- /dev/null +++ b/pkg/sdkserver/datasets.go @@ -0,0 +1,329 @@ +package sdkserver + +import ( + "encoding/json" + "fmt" + "net/http" + + gcontext "github.com/gptscript-ai/gptscript/pkg/context" + "github.com/gptscript-ai/gptscript/pkg/gptscript" + "github.com/gptscript-ai/gptscript/pkg/loader" +) + +type datasetRequest struct { + Input string `json:"input"` + Workspace string `json:"workspace"` + DatasetToolRepo string `json:"datasetToolRepo"` +} + +func (r datasetRequest) validate(requireInput bool) error { + if r.Workspace == "" { + return fmt.Errorf("workspace is required") + } else if requireInput && r.Input == "" { + return fmt.Errorf("input is required") + } + return nil +} + +func (r datasetRequest) opts(o gptscript.Options) gptscript.Options { + opts := gptscript.Options{ + Cache: o.Cache, + Monitor: o.Monitor, + Runner: o.Runner, + DatasetToolRepo: o.DatasetToolRepo, + Workspace: r.Workspace, + } + if r.DatasetToolRepo != "" { + opts.DatasetToolRepo = r.DatasetToolRepo + } + return opts +} + +func (s *server) listDatasets(w http.ResponseWriter, r *http.Request) { + logger := gcontext.GetLogger(r.Context()) + + var req datasetRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(logger, w, http.StatusBadRequest, fmt.Errorf("failed to decode request body: %w", err)) + return + } + + if err := req.validate(false); err != nil { + writeError(logger, w, http.StatusBadRequest, err) + return + } + + g, err := gptscript.New(r.Context(), req.opts(s.gptscriptOpts)) + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to initialize gptscript: %w", err)) + return + } + + prg, err := loader.Program(r.Context(), "List Datasets from "+s.gptscriptOpts.DatasetToolRepo, "", loader.Options{ + Cache: g.Cache, + }) + + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to load program: %w", err)) + return + } + + result, err := g.Run(r.Context(), prg, s.gptscriptOpts.Env, req.Input) + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to run program: %w", err)) + return + } + + writeResponse(logger, w, result) +} + +type createDatasetArgs struct { + Name string `json:"dataset_name"` + Description string `json:"dataset_description"` +} + +func (a createDatasetArgs) validate() error { + if a.Name == "" { + return fmt.Errorf("dataset_name is required") + } + return nil +} + +func (s *server) createDataset(w http.ResponseWriter, r *http.Request) { + logger := gcontext.GetLogger(r.Context()) + + var req datasetRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(logger, w, http.StatusBadRequest, fmt.Errorf("failed to decode request body: %w", err)) + return + } + + if err := req.validate(true); err != nil { + writeError(logger, w, http.StatusBadRequest, err) + return + } + + g, err := gptscript.New(r.Context(), req.opts(s.gptscriptOpts)) + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to initialize gptscript: %w", err)) + return + } + + var args createDatasetArgs + if err := json.Unmarshal([]byte(req.Input), &args); err != nil { + writeError(logger, w, http.StatusBadRequest, fmt.Errorf("failed to unmarshal input: %w", err)) + return + } + + if err := args.validate(); err != nil { + writeError(logger, w, http.StatusBadRequest, err) + return + } + + prg, err := loader.Program(r.Context(), "Create Dataset from "+s.gptscriptOpts.DatasetToolRepo, "", loader.Options{ + Cache: g.Cache, + }) + + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to load program: %w", err)) + return + } + + result, err := g.Run(r.Context(), prg, s.gptscriptOpts.Env, req.Input) + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to run program: %w", err)) + return + } + + writeResponse(logger, w, result) +} + +type addDatasetElementArgs struct { + DatasetID string `json:"dataset_id"` + ElementName string `json:"element_name"` + ElementDescription string `json:"element_description"` + ElementContent string `json:"element_content"` +} + +func (a addDatasetElementArgs) validate() error { + if a.DatasetID == "" { + return fmt.Errorf("dataset_id is required") + } + if a.ElementName == "" { + return fmt.Errorf("element_name is required") + } + if a.ElementContent == "" { + return fmt.Errorf("element_content is required") + } + return nil +} + +func (s *server) addDatasetElement(w http.ResponseWriter, r *http.Request) { + logger := gcontext.GetLogger(r.Context()) + + var req datasetRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(logger, w, http.StatusBadRequest, fmt.Errorf("failed to decode request body: %w", err)) + return + } + + if err := req.validate(true); err != nil { + writeError(logger, w, http.StatusBadRequest, err) + return + } + + g, err := gptscript.New(r.Context(), req.opts(s.gptscriptOpts)) + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to initialize gptscript: %w", err)) + return + } + + var args addDatasetElementArgs + if err := json.Unmarshal([]byte(req.Input), &args); err != nil { + writeError(logger, w, http.StatusBadRequest, fmt.Errorf("failed to unmarshal input: %w", err)) + return + } + + if err := args.validate(); err != nil { + writeError(logger, w, http.StatusBadRequest, err) + return + } + + prg, err := loader.Program(r.Context(), "Add Element from "+s.gptscriptOpts.DatasetToolRepo, "", loader.Options{ + Cache: g.Cache, + }) + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to load program: %w", err)) + return + } + + result, err := g.Run(r.Context(), prg, s.gptscriptOpts.Env, req.Input) + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to run program: %w", err)) + return + } + + writeResponse(logger, w, result) +} + +type listDatasetElementsArgs struct { + DatasetID string `json:"dataset_id"` +} + +func (a listDatasetElementsArgs) validate() error { + if a.DatasetID == "" { + return fmt.Errorf("dataset_id is required") + } + return nil +} + +func (s *server) listDatasetElements(w http.ResponseWriter, r *http.Request) { + logger := gcontext.GetLogger(r.Context()) + + var req datasetRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(logger, w, http.StatusBadRequest, fmt.Errorf("failed to decode request body: %w", err)) + return + } + + if err := req.validate(true); err != nil { + writeError(logger, w, http.StatusBadRequest, err) + return + } + + g, err := gptscript.New(r.Context(), req.opts(s.gptscriptOpts)) + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to initialize gptscript: %w", err)) + return + } + + var args listDatasetElementsArgs + if err := json.Unmarshal([]byte(req.Input), &args); err != nil { + writeError(logger, w, http.StatusBadRequest, fmt.Errorf("failed to unmarshal input: %w", err)) + return + } + + if err := args.validate(); err != nil { + writeError(logger, w, http.StatusBadRequest, err) + return + } + + prg, err := loader.Program(r.Context(), "List Elements from "+s.gptscriptOpts.DatasetToolRepo, "", loader.Options{ + Cache: g.Cache, + }) + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to load program: %w", err)) + return + } + + result, err := g.Run(r.Context(), prg, s.gptscriptOpts.Env, req.Input) + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to run program: %w", err)) + return + } + + writeResponse(logger, w, result) +} + +type getDatasetElementArgs struct { + DatasetID string `json:"dataset_id"` + Element string `json:"element"` +} + +func (a getDatasetElementArgs) validate() error { + if a.DatasetID == "" { + return fmt.Errorf("dataset_id is required") + } + if a.Element == "" { + return fmt.Errorf("element is required") + } + return nil +} + +func (s *server) getDatasetElement(w http.ResponseWriter, r *http.Request) { + logger := gcontext.GetLogger(r.Context()) + + var req datasetRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(logger, w, http.StatusBadRequest, fmt.Errorf("failed to decode request body: %w", err)) + return + } + + if err := req.validate(true); err != nil { + writeError(logger, w, http.StatusBadRequest, err) + return + } + + g, err := gptscript.New(r.Context(), req.opts(s.gptscriptOpts)) + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to initialize gptscript: %w", err)) + return + } + + var args getDatasetElementArgs + if err := json.Unmarshal([]byte(req.Input), &args); err != nil { + writeError(logger, w, http.StatusBadRequest, fmt.Errorf("failed to unmarshal input: %w", err)) + return + } + + if err := args.validate(); err != nil { + writeError(logger, w, http.StatusBadRequest, err) + return + } + + prg, err := loader.Program(r.Context(), "Get Element from "+s.gptscriptOpts.DatasetToolRepo, "", loader.Options{ + Cache: g.Cache, + }) + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to load program: %w", err)) + return + } + + result, err := g.Run(r.Context(), prg, s.gptscriptOpts.Env, req.Input) + if err != nil { + writeError(logger, w, http.StatusInternalServerError, fmt.Errorf("failed to run program: %w", err)) + return + } + + writeResponse(logger, w, result) +} diff --git a/pkg/sdkserver/routes.go b/pkg/sdkserver/routes.go index fc69a08c..8427a6a5 100644 --- a/pkg/sdkserver/routes.go +++ b/pkg/sdkserver/routes.go @@ -66,6 +66,12 @@ func (s *server) addRoutes(mux *http.ServeMux) { mux.HandleFunc("POST /credentials/create", s.createCredential) mux.HandleFunc("POST /credentials/reveal", s.revealCredential) mux.HandleFunc("POST /credentials/delete", s.deleteCredential) + + mux.HandleFunc("POST /datasets", s.listDatasets) + mux.HandleFunc("POST /datasets/create", s.createDataset) + mux.HandleFunc("POST /datasets/list-elements", s.listDatasetElements) + mux.HandleFunc("POST /datasets/get-element", s.getDatasetElement) + mux.HandleFunc("POST /datasets/add-element", s.addDatasetElement) } // health just provides an endpoint for checking whether the server is running and accessible. From 9d50b05e379d4cde1377d446ea4c24d93291d65d Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Wed, 9 Oct 2024 23:23:38 -0400 Subject: [PATCH 2/9] return as stdout Signed-off-by: Grant Linville --- pkg/sdkserver/datasets.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/sdkserver/datasets.go b/pkg/sdkserver/datasets.go index 2f40971a..3c9487da 100644 --- a/pkg/sdkserver/datasets.go +++ b/pkg/sdkserver/datasets.go @@ -74,7 +74,7 @@ func (s *server) listDatasets(w http.ResponseWriter, r *http.Request) { return } - writeResponse(logger, w, result) + writeResponse(logger, w, map[string]any{"stdout": result}) } type createDatasetArgs struct { @@ -135,7 +135,7 @@ func (s *server) createDataset(w http.ResponseWriter, r *http.Request) { return } - writeResponse(logger, w, result) + writeResponse(logger, w, map[string]any{"stdout": result}) } type addDatasetElementArgs struct { @@ -203,7 +203,7 @@ func (s *server) addDatasetElement(w http.ResponseWriter, r *http.Request) { return } - writeResponse(logger, w, result) + writeResponse(logger, w, map[string]any{"stdout": result}) } type listDatasetElementsArgs struct { @@ -262,7 +262,7 @@ func (s *server) listDatasetElements(w http.ResponseWriter, r *http.Request) { return } - writeResponse(logger, w, result) + writeResponse(logger, w, map[string]any{"stdout": result}) } type getDatasetElementArgs struct { @@ -325,5 +325,5 @@ func (s *server) getDatasetElement(w http.ResponseWriter, r *http.Request) { return } - writeResponse(logger, w, result) + writeResponse(logger, w, map[string]any{"stdout": result}) } From 2a63506148367a0a7bbda695e6750c01c2c6d16d Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Wed, 9 Oct 2024 23:34:06 -0400 Subject: [PATCH 3/9] update generated docs Signed-off-by: Grant Linville --- docs/docs/04-command-line-reference/gptscript.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docs/04-command-line-reference/gptscript.md b/docs/docs/04-command-line-reference/gptscript.md index 8a726c64..9c7fb059 100644 --- a/docs/docs/04-command-line-reference/gptscript.md +++ b/docs/docs/04-command-line-reference/gptscript.md @@ -20,6 +20,7 @@ gptscript [flags] PROGRAM_FILE [INPUT...] --confirm Prompt before running potentially dangerous commands ($GPTSCRIPT_CONFIRM) --credential-context strings Context name(s) in which to store credentials ($GPTSCRIPT_CREDENTIAL_CONTEXT) --credential-override strings Credentials to override (ex: --credential-override github.com/example/cred-tool:API_TOKEN=1234) ($GPTSCRIPT_CREDENTIAL_OVERRIDE) + --dataset-tool-repo string The repo to use for dataset tools ($GPTSCRIPT_DATASET_TOOL_REPO) (default "github.com/gptscript-ai/datasets") --debug Enable debug logging ($GPTSCRIPT_DEBUG) --debug-messages Enable logging of chat completion calls ($GPTSCRIPT_DEBUG_MESSAGES) --default-model string Default LLM model to use ($GPTSCRIPT_DEFAULT_MODEL) (default "gpt-4o") From 955ad9c2ee25fe2838a9f82e5b2ddbffd9df1833 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Thu, 10 Oct 2024 11:57:29 -0400 Subject: [PATCH 4/9] PR feedback Signed-off-by: Grant Linville --- pkg/gptscript/gptscript.go | 5 ----- pkg/sdkserver/server.go | 6 ++++++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/gptscript/gptscript.go b/pkg/gptscript/gptscript.go index a6bff8f5..dda2ab75 100644 --- a/pkg/gptscript/gptscript.go +++ b/pkg/gptscript/gptscript.go @@ -32,8 +32,6 @@ import ( var log = mvl.Package() -const defaultDatasetToolRepo = "github.com/gptscript-ai/datasets" - type GPTScript struct { Registry *llm.Registry Runner *runner.Runner @@ -84,9 +82,6 @@ func Complete(opts ...Options) Options { if len(result.CredentialContexts) == 0 { result.CredentialContexts = []string{credentials.DefaultCredentialContext} } - if result.DatasetToolRepo == "" { - result.DatasetToolRepo = defaultDatasetToolRepo - } return result } diff --git a/pkg/sdkserver/server.go b/pkg/sdkserver/server.go index 0a68f0fa..087dc68c 100644 --- a/pkg/sdkserver/server.go +++ b/pkg/sdkserver/server.go @@ -23,6 +23,8 @@ import ( "github.com/rs/cors" ) +const defaultDatasetToolRepo = "github.com/gptscript-ai/datasets" + type Options struct { gptscript.Options @@ -165,5 +167,9 @@ func complete(opts ...Options) Options { result.ListenAddress = "127.0.0.1:0" } + if result.DatasetToolRepo == "" { + result.DatasetToolRepo = defaultDatasetToolRepo + } + return result } From 175a8e358e45f0db74a470676ae022e536fe89d7 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Thu, 10 Oct 2024 12:43:25 -0400 Subject: [PATCH 5/9] remove dataset tool repo as a GPTScript option Signed-off-by: Grant Linville --- pkg/cli/gptscript.go | 2 -- pkg/gptscript/gptscript.go | 2 -- pkg/sdkserver/datasets.go | 27 +++++++++++++++------------ pkg/sdkserver/server.go | 6 ------ 4 files changed, 15 insertions(+), 22 deletions(-) diff --git a/pkg/cli/gptscript.go b/pkg/cli/gptscript.go index 19ab7b29..66719adc 100644 --- a/pkg/cli/gptscript.go +++ b/pkg/cli/gptscript.go @@ -75,7 +75,6 @@ type GPTScript struct { SaveChatStateFile string `usage:"A file to save the chat state to so that a conversation can be resumed with --chat-state" local:"true"` DefaultModelProvider string `usage:"Default LLM model provider to use, this will override OpenAI settings"` GithubEnterpriseHostname string `usage:"The host name for a Github Enterprise instance to enable for remote loading" local:"true"` - DatasetToolRepo string `usage:"The repo to use for dataset tools" default:"github.com/gptscript-ai/datasets" local:"true"` readData []byte } @@ -147,7 +146,6 @@ func (r *GPTScript) NewGPTScriptOpts() (gptscript.Options, error) { Workspace: r.Workspace, DisablePromptServer: r.UI, DefaultModelProvider: r.DefaultModelProvider, - DatasetToolRepo: r.DatasetToolRepo, } if r.Confirm { diff --git a/pkg/gptscript/gptscript.go b/pkg/gptscript/gptscript.go index dda2ab75..679eb503 100644 --- a/pkg/gptscript/gptscript.go +++ b/pkg/gptscript/gptscript.go @@ -51,7 +51,6 @@ type Options struct { CredentialContexts []string Quiet *bool Workspace string - DatasetToolRepo string DisablePromptServer bool Env []string } @@ -67,7 +66,6 @@ func Complete(opts ...Options) Options { result.CredentialContexts = opt.CredentialContexts result.Quiet = types.FirstSet(opt.Quiet, result.Quiet) result.Workspace = types.FirstSet(opt.Workspace, result.Workspace) - result.DatasetToolRepo = types.FirstSet(opt.DatasetToolRepo, result.DatasetToolRepo) result.Env = append(result.Env, opt.Env...) result.DisablePromptServer = types.FirstSet(opt.DisablePromptServer, result.DisablePromptServer) result.DefaultModelProvider = types.FirstSet(opt.DefaultModelProvider, result.DefaultModelProvider) diff --git a/pkg/sdkserver/datasets.go b/pkg/sdkserver/datasets.go index 3c9487da..a6372b39 100644 --- a/pkg/sdkserver/datasets.go +++ b/pkg/sdkserver/datasets.go @@ -27,16 +27,19 @@ func (r datasetRequest) validate(requireInput bool) error { func (r datasetRequest) opts(o gptscript.Options) gptscript.Options { opts := gptscript.Options{ - Cache: o.Cache, - Monitor: o.Monitor, - Runner: o.Runner, - DatasetToolRepo: o.DatasetToolRepo, - Workspace: r.Workspace, + Cache: o.Cache, + Monitor: o.Monitor, + Runner: o.Runner, + Workspace: r.Workspace, } + return opts +} + +func (r datasetRequest) getToolRepo() string { if r.DatasetToolRepo != "" { - opts.DatasetToolRepo = r.DatasetToolRepo + return r.DatasetToolRepo } - return opts + return "github.com/gptscript-ai/datasets" } func (s *server) listDatasets(w http.ResponseWriter, r *http.Request) { @@ -59,7 +62,7 @@ func (s *server) listDatasets(w http.ResponseWriter, r *http.Request) { return } - prg, err := loader.Program(r.Context(), "List Datasets from "+s.gptscriptOpts.DatasetToolRepo, "", loader.Options{ + prg, err := loader.Program(r.Context(), "List Datasets from "+req.getToolRepo(), "", loader.Options{ Cache: g.Cache, }) @@ -120,7 +123,7 @@ func (s *server) createDataset(w http.ResponseWriter, r *http.Request) { return } - prg, err := loader.Program(r.Context(), "Create Dataset from "+s.gptscriptOpts.DatasetToolRepo, "", loader.Options{ + prg, err := loader.Program(r.Context(), "Create Dataset from "+req.getToolRepo(), "", loader.Options{ Cache: g.Cache, }) @@ -189,7 +192,7 @@ func (s *server) addDatasetElement(w http.ResponseWriter, r *http.Request) { return } - prg, err := loader.Program(r.Context(), "Add Element from "+s.gptscriptOpts.DatasetToolRepo, "", loader.Options{ + prg, err := loader.Program(r.Context(), "Add Element from "+req.getToolRepo(), "", loader.Options{ Cache: g.Cache, }) if err != nil { @@ -248,7 +251,7 @@ func (s *server) listDatasetElements(w http.ResponseWriter, r *http.Request) { return } - prg, err := loader.Program(r.Context(), "List Elements from "+s.gptscriptOpts.DatasetToolRepo, "", loader.Options{ + prg, err := loader.Program(r.Context(), "List Elements from "+req.getToolRepo(), "", loader.Options{ Cache: g.Cache, }) if err != nil { @@ -311,7 +314,7 @@ func (s *server) getDatasetElement(w http.ResponseWriter, r *http.Request) { return } - prg, err := loader.Program(r.Context(), "Get Element from "+s.gptscriptOpts.DatasetToolRepo, "", loader.Options{ + prg, err := loader.Program(r.Context(), "Get Element from "+req.getToolRepo(), "", loader.Options{ Cache: g.Cache, }) if err != nil { diff --git a/pkg/sdkserver/server.go b/pkg/sdkserver/server.go index 087dc68c..0a68f0fa 100644 --- a/pkg/sdkserver/server.go +++ b/pkg/sdkserver/server.go @@ -23,8 +23,6 @@ import ( "github.com/rs/cors" ) -const defaultDatasetToolRepo = "github.com/gptscript-ai/datasets" - type Options struct { gptscript.Options @@ -167,9 +165,5 @@ func complete(opts ...Options) Options { result.ListenAddress = "127.0.0.1:0" } - if result.DatasetToolRepo == "" { - result.DatasetToolRepo = defaultDatasetToolRepo - } - return result } From f11feeda53982a920a645c849cf7d929c4d197e4 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Thu, 10 Oct 2024 12:47:11 -0400 Subject: [PATCH 6/9] update generated docs Signed-off-by: Grant Linville --- docs/docs/04-command-line-reference/gptscript.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/docs/04-command-line-reference/gptscript.md b/docs/docs/04-command-line-reference/gptscript.md index 9c7fb059..8a726c64 100644 --- a/docs/docs/04-command-line-reference/gptscript.md +++ b/docs/docs/04-command-line-reference/gptscript.md @@ -20,7 +20,6 @@ gptscript [flags] PROGRAM_FILE [INPUT...] --confirm Prompt before running potentially dangerous commands ($GPTSCRIPT_CONFIRM) --credential-context strings Context name(s) in which to store credentials ($GPTSCRIPT_CREDENTIAL_CONTEXT) --credential-override strings Credentials to override (ex: --credential-override github.com/example/cred-tool:API_TOKEN=1234) ($GPTSCRIPT_CREDENTIAL_OVERRIDE) - --dataset-tool-repo string The repo to use for dataset tools ($GPTSCRIPT_DATASET_TOOL_REPO) (default "github.com/gptscript-ai/datasets") --debug Enable debug logging ($GPTSCRIPT_DEBUG) --debug-messages Enable logging of chat completion calls ($GPTSCRIPT_DEBUG_MESSAGES) --default-model string Default LLM model to use ($GPTSCRIPT_DEFAULT_MODEL) (default "gpt-4o") From 3e784101b5a5ad4adc266a703507c6673103044c Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Thu, 10 Oct 2024 14:08:06 -0400 Subject: [PATCH 7/9] use snake case Signed-off-by: Grant Linville --- pkg/sdkserver/datasets.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sdkserver/datasets.go b/pkg/sdkserver/datasets.go index a6372b39..34d45f35 100644 --- a/pkg/sdkserver/datasets.go +++ b/pkg/sdkserver/datasets.go @@ -13,7 +13,7 @@ import ( type datasetRequest struct { Input string `json:"input"` Workspace string `json:"workspace"` - DatasetToolRepo string `json:"datasetToolRepo"` + DatasetToolRepo string `json:"dataset_tool_repo"` } func (r datasetRequest) validate(requireInput bool) error { From bcc1f221badb740a86f87ef3dfcb0eaa6def90ef Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Thu, 10 Oct 2024 16:05:13 -0400 Subject: [PATCH 8/9] use camelCase Signed-off-by: Grant Linville --- pkg/sdkserver/datasets.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/sdkserver/datasets.go b/pkg/sdkserver/datasets.go index 34d45f35..5863299d 100644 --- a/pkg/sdkserver/datasets.go +++ b/pkg/sdkserver/datasets.go @@ -13,7 +13,7 @@ import ( type datasetRequest struct { Input string `json:"input"` Workspace string `json:"workspace"` - DatasetToolRepo string `json:"dataset_tool_repo"` + DatasetToolRepo string `json:"datasetToolRepo"` } func (r datasetRequest) validate(requireInput bool) error { @@ -81,8 +81,8 @@ func (s *server) listDatasets(w http.ResponseWriter, r *http.Request) { } type createDatasetArgs struct { - Name string `json:"dataset_name"` - Description string `json:"dataset_description"` + Name string `json:"datasetName"` + Description string `json:"datasetDescription"` } func (a createDatasetArgs) validate() error { @@ -142,10 +142,10 @@ func (s *server) createDataset(w http.ResponseWriter, r *http.Request) { } type addDatasetElementArgs struct { - DatasetID string `json:"dataset_id"` - ElementName string `json:"element_name"` - ElementDescription string `json:"element_description"` - ElementContent string `json:"element_content"` + DatasetID string `json:"datasetID"` + ElementName string `json:"elementName"` + ElementDescription string `json:"elementDescription"` + ElementContent string `json:"elementContent"` } func (a addDatasetElementArgs) validate() error { @@ -210,7 +210,7 @@ func (s *server) addDatasetElement(w http.ResponseWriter, r *http.Request) { } type listDatasetElementsArgs struct { - DatasetID string `json:"dataset_id"` + DatasetID string `json:"datasetID"` } func (a listDatasetElementsArgs) validate() error { @@ -269,7 +269,7 @@ func (s *server) listDatasetElements(w http.ResponseWriter, r *http.Request) { } type getDatasetElementArgs struct { - DatasetID string `json:"dataset_id"` + DatasetID string `json:"datasetID"` Element string `json:"element"` } From 6c5796efd97a8b7cdf09184727738fe34a6f89af Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Thu, 10 Oct 2024 16:23:09 -0400 Subject: [PATCH 9/9] update error messages Signed-off-by: Grant Linville --- pkg/sdkserver/datasets.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/sdkserver/datasets.go b/pkg/sdkserver/datasets.go index 5863299d..0085132c 100644 --- a/pkg/sdkserver/datasets.go +++ b/pkg/sdkserver/datasets.go @@ -87,7 +87,7 @@ type createDatasetArgs struct { func (a createDatasetArgs) validate() error { if a.Name == "" { - return fmt.Errorf("dataset_name is required") + return fmt.Errorf("datasetName is required") } return nil } @@ -150,13 +150,13 @@ type addDatasetElementArgs struct { func (a addDatasetElementArgs) validate() error { if a.DatasetID == "" { - return fmt.Errorf("dataset_id is required") + return fmt.Errorf("datasetID is required") } if a.ElementName == "" { - return fmt.Errorf("element_name is required") + return fmt.Errorf("elementName is required") } if a.ElementContent == "" { - return fmt.Errorf("element_content is required") + return fmt.Errorf("elementContent is required") } return nil } @@ -215,7 +215,7 @@ type listDatasetElementsArgs struct { func (a listDatasetElementsArgs) validate() error { if a.DatasetID == "" { - return fmt.Errorf("dataset_id is required") + return fmt.Errorf("datasetID is required") } return nil } @@ -275,7 +275,7 @@ type getDatasetElementArgs struct { func (a getDatasetElementArgs) validate() error { if a.DatasetID == "" { - return fmt.Errorf("dataset_id is required") + return fmt.Errorf("datasetID is required") } if a.Element == "" { return fmt.Errorf("element is required")