Refactor User manager service (Velocidex#3117)

Split off storage with ACL management. Added TTL cache for user records and gui options and setup TTL expiration processing.
Genio40921 · Nov 22, 2023 · 1abb528 · 1abb528
1 parent 51f2d34
commit 1abb528
Show file tree

Hide file tree

Showing 55 changed files with 1,081 additions and 753 deletions.
diff --git a/api/authenticators/basic.go b/api/authenticators/basic.go
@@ -12,7 +12,6 @@ import (
 	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/services"
-	"www.velocidex.com/golang/velociraptor/services/users"
 )
 
 // Implement basic authentication.
@@ -81,7 +80,8 @@ func (self *BasicAuthenticator) AuthenticateUserHandler(
 		// Get the full user record with hashes so we can
 		// verify it below.
 		users_manager := services.GetUserManager()
-		user_record, err := users_manager.GetUserWithHashes(r.Context(), username)
+		user_record, err := users_manager.GetUserWithHashes(r.Context(),
+			username, username)
 		if err != nil || user_record.Name != username {
 			services.LogAudit(r.Context(),
 				self.config_obj, username, "Unknown username",
@@ -92,12 +92,14 @@ func (self *BasicAuthenticator) AuthenticateUserHandler(
 			http.Error(w, "authorization failed", http.StatusUnauthorized)
 			return
 		}
-
-		if !users.VerifyPassword(user_record, password) {
+		ok, err = users_manager.VerifyPassword(r.Context(),
+			username, username, password)
+		if !ok || err != nil {
 			services.LogAudit(r.Context(),
 				self.config_obj, username, "Invalid password",
 				ordereddict.NewDict().
 					Set("remote", r.RemoteAddr).
+					Set("error", err.Error()).
 					Set("status", http.StatusUnauthorized))
 
 			http.Error(w, "authorization failed", http.StatusUnauthorized)

diff --git a/api/authenticators/certs.go b/api/authenticators/certs.go
@@ -89,7 +89,6 @@ import (
 	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/services"
-	"www.velocidex.com/golang/velociraptor/users"
 )
 
 var (
@@ -166,7 +165,7 @@ func (self *CertAuthenticator) AuthenticateUserHandler(
 		}
 
 		users_manager := services.GetUserManager()
-		user_record, err := users_manager.GetUser(r.Context(), username)
+		user_record, err := users_manager.GetUser(r.Context(), username, username)
 		if err != nil {
 			if err != services.UserNotFoundError || len(self.default_roles) == 0 {
 				http.Error(w,
@@ -187,7 +186,7 @@ func (self *CertAuthenticator) AuthenticateUserHandler(
 
 			// Use the super user principal to actually add the
 			// username so we have enough permissions.
-			err = users.AddUserToOrg(r.Context(), users.AddNewUser,
+			err = users_manager.AddUserToOrg(r.Context(), services.AddNewUser,
 				constants.PinnedServerName, username,
 				[]string{"root"}, policy)
 			if err != nil {

diff --git a/api/authenticators/google.go b/api/authenticators/google.go
@@ -288,7 +288,7 @@ func authenticateUserHandle(
 
 		// Now check if the user is allowed to log in.
 		users := services.GetUserManager()
-		user_record, err := users.GetUser(r.Context(), username)
+		user_record, err := users.GetUser(r.Context(), username, username)
 		if err != nil || user_record.Name != username {
 			reject_cb(w, r, errors.New("Invalid user"), username)
 			return

diff --git a/api/authenticators/saml.go b/api/authenticators/saml.go
@@ -112,7 +112,7 @@ func (self *SamlAuthenticator) AuthenticateUserHandler(
 
 		username := sa.GetAttributes().Get(self.user_attribute)
 		users := services.GetUserManager()
-		user_record, err := users.GetUser(r.Context(), username)
+		user_record, err := users.GetUser(r.Context(), username, username)
 		if err == nil && user_record.Name == username {
 			// Does the user have access to the specified org?
 			err = CheckOrgAccess(r, user_record)

diff --git a/api/users.go b/api/users.go
@@ -13,7 +13,6 @@ import (
 	acl_proto "www.velocidex.com/golang/velociraptor/acls/proto"
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
 	"www.velocidex.com/golang/velociraptor/services"
-	"www.velocidex.com/golang/velociraptor/users"
 )
 
 // This is only used to set the user's own password which is always
@@ -50,7 +49,7 @@ func (self *ApiServer) SetPassword(
 		target = principal
 	}
 
-	err = users.SetUserPassword(
+	err = user_manager.SetUserPassword(
 		ctx, org_config_obj, principal, target, in.Password, "")
 	if err != nil {
 		return nil, Status(self.verbose, err)
@@ -72,7 +71,7 @@ func (self *ApiServer) GetUsers(
 	principal := user_record.Name
 
 	// Only show users in the current org
-	users, err := users.ListUsers(ctx, principal, []string{org_config_obj.OrgId})
+	users, err := user_manager.ListUsers(ctx, principal, []string{org_config_obj.OrgId})
 	if err != nil {
 		return nil, Status(self.verbose, err)
 	}
@@ -94,7 +93,7 @@ func (self *ApiServer) GetGlobalUsers(
 	principal := user_record.Name
 
 	// Show all users visible to us
-	users, err := users.ListUsers(ctx, principal, []string{})
+	users, err := user_manager.ListUsers(ctx, principal, []string{})
 	if err != nil {
 		return nil, Status(self.verbose, err)
 	}
@@ -120,12 +119,12 @@ func (self *ApiServer) CreateUser(ctx context.Context,
 		Roles: in.Roles,
 	}
 
-	mode := users.UseExistingUser
+	mode := services.UseExistingUser
 	if in.AddNewUser {
-		mode = users.AddNewUser
+		mode = services.AddNewUser
 	}
 
-	err = users.AddUserToOrg(ctx, mode, principal, in.Name, in.Orgs, acl)
+	err = users_manager.AddUserToOrg(ctx, mode, principal, in.Name, in.Orgs, acl)
 
 	if err == nil {
 		services.LogAudit(ctx,
@@ -148,7 +147,7 @@ func (self *ApiServer) GetUser(
 		return nil, err
 	}
 
-	user, err := users.GetUser(ctx, user_record.Name, in.Name)
+	user, err := users_manager.GetUser(ctx, user_record.Name, in.Name)
 	if err != nil {
 		if errors.Is(err, acls.PermissionDenied) {
 			return nil, status.Error(codes.PermissionDenied,
@@ -255,7 +254,7 @@ func (self *ApiServer) SetUserRoles(
 
 	// Now attempt to set the ACL - permission checks are done by
 	// users.AddUserToOrg
-	err = users.AddUserToOrg(ctx, users.UseExistingUser,
+	err = users_manager.AddUserToOrg(ctx, services.UseExistingUser,
 		principal, in.Name, []string{in.Org}, acl)
 
 	if err == nil {

diff --git a/artifacts/definitions/Server/Internal/UserManager.yaml b/artifacts/definitions/Server/Internal/UserManager.yaml
@@ -0,0 +1,4 @@
+name: Server.Internal.UserManager
+type: INTERNAL
+description: |
+  An internal artifact notifying when user accounts are modified.
diff --git a/artifacts/testdata/server/testcases/orgs.out.yaml b/artifacts/testdata/server/testcases/orgs.out.yaml
@@ -59,16 +59,6 @@ SELECT Name, OrgId FROM orgs() ORDER BY OrgId[
    "delete_results": true
   }
  },
- {
-  "name": "OrgAdmin",
-  "org_id": "ORGID",
-  "org_name": "MyOrg",
-  "picture": "",
-  "email": false,
-  "roles": null,
-  "_policy": {},
-  "effective_policy": null
- },
  {
   "name": "OrgUser",
   "org_id": "ORGID",
@@ -196,16 +186,6 @@ SELECT Name, OrgId FROM orgs() ORDER BY OrgId[
   }
  }
 ]SELECT * FROM query(query={ SELECT * FROM gui_users() ORDER BY name }, org_id="ORGID")[
- {
-  "name": "OrgAdmin",
-  "org_id": "ORGID",
-  "org_name": "MyOrg",
-  "picture": "",
-  "email": false,
-  "roles": null,
-  "_policy": {},
-  "effective_policy": null
- },
  {
   "name": "OrgUser",
   "org_id": "ORGID",

diff --git a/artifacts/testdata/server/testcases/users.out.yaml b/artifacts/testdata/server/testcases/users.out.yaml
@@ -172,6 +172,6 @@ SELECT whoami() FROM scope()[
  }
 ]SELECT * FROM test_read_logs() WHERE Log =~ "Username is reserved" AND NOT Log =~ "SELECT"[
  {
-  "Log": "Velociraptor: user_create: Username is reserved\n"
+  "Log": "Velociraptor: user_create: Username is reserved: VelociraptorServer\n"
  }
 ]
diff --git a/bin/config.go b/bin/config.go
@@ -34,6 +34,7 @@ import (
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
 	"www.velocidex.com/golang/velociraptor/config"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/crypto"
 	"www.velocidex.com/golang/velociraptor/crypto/utils"
 	"www.velocidex.com/golang/velociraptor/json"
@@ -550,7 +551,8 @@ func doDumpApiClientConfig() error {
 
 		// Make sure the user actually exists.
 		user_manager := services.GetUserManager()
-		_, err = user_manager.GetUser(ctx, *config_api_client_common_name)
+		_, err = user_manager.GetUser(ctx, constants.PinnedServerName,
+			*config_api_client_common_name)
 		if err != nil {
 			// Need to ensure we have a user
 			err := user_manager.SetUser(ctx, &api_proto.VelociraptorUser{

diff --git a/bin/grant.go b/bin/grant.go
@@ -8,6 +8,7 @@ import (
 	jsonpatch "github.com/evanphx/json-patch/v5"
 	"www.velocidex.com/golang/velociraptor/acls"
 	acl_proto "www.velocidex.com/golang/velociraptor/acls/proto"
+	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/json"
 	logging "www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/services"
@@ -78,7 +79,7 @@ func doGrant() error {
 
 	// Check the user actually exists first
 	user_manager := services.GetUserManager()
-	_, err = user_manager.GetUser(ctx, principal)
+	_, err = user_manager.GetUser(ctx, constants.PinnedServerName, principal)
 	if err != nil {
 		return err
 	}

diff --git a/bin/orgs.go b/bin/orgs.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
+	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/json"
 	logging "www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/services"
@@ -87,7 +88,7 @@ func doOrgUserAdd() error {
 
 	user_manager := services.GetUserManager()
 	record, err := user_manager.GetUserWithHashes(
-		ctx, *orgs_user_add_user)
+		ctx, constants.PinnedServerName, *orgs_user_add_user)
 	if err != nil {
 		return err
 	}
@@ -179,7 +180,8 @@ func doOrgDelete() error {
 	logger := logging.GetLogger(config_obj, &logging.ToolComponent)
 	logger.Info("Will remove org %v\n", *orgs_delete_org_id)
 
-	return org_manager.DeleteOrg(ctx, *orgs_delete_org_id)
+	return org_manager.DeleteOrg(ctx,
+		constants.PinnedServerName, *orgs_delete_org_id)
 }
 
 func init() {

diff --git a/bin/users.go b/bin/users.go
@@ -25,6 +25,7 @@ import (
 
 	"golang.org/x/crypto/ssh/terminal"
 	"www.velocidex.com/golang/velociraptor/api/authenticators"
+	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/json"
 	logging "www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/services"
@@ -152,13 +153,15 @@ func doShowUser() error {
 	defer sm.Close()
 
 	users_manager := services.GetUserManager()
-	user_record, err := users_manager.GetUser(ctx, *user_show_name)
+	user_record, err := users_manager.GetUser(ctx, constants.PinnedServerName,
+		*user_show_name)
 	if err != nil {
 		return err
 	}
 
 	if *user_show_hashes {
-		user_record, err := users_manager.GetUserWithHashes(ctx, *user_show_name)
+		user_record, err := users_manager.GetUserWithHashes(ctx,
+			constants.PinnedServerName, *user_show_name)
 		if err != nil {
 			return err
 		}
@@ -197,7 +200,8 @@ func doLockUser() error {
 	defer sm.Close()
 
 	users_manager := services.GetUserManager()
-	user_record, err := users_manager.GetUser(ctx, *user_lock_name)
+	user_record, err := users_manager.GetUser(ctx, constants.PinnedServerName,
+		*user_lock_name)
 	if err != nil {
 		return fmt.Errorf("Unable to find user %s", *user_lock_name)
 	}

diff --git a/constants/constants.go b/constants/constants.go
@@ -113,6 +113,7 @@ const (
 	SQLITE_ALWAYS_MAKE_TEMPFILE = "SQLITE_ALWAYS_MAKE_TEMPFILE"
 
 	PinnedServerName = "VelociraptorServer"
+	PinnedGwName     = "GRPC_GW"
 
 	CLIENT_API_VERSION = uint32(4)
 

diff --git a/json/wrappers.go b/json/wrappers.go
@@ -125,7 +125,10 @@ func Unmarshal(b []byte, v interface{}) error {
 
 	self, ok := v.(proto.Message)
 	if ok {
-		return protojson.Unmarshal(b, self)
+		options := &protojson.UnmarshalOptions{
+			DiscardUnknown: true,
+		}
+		return options.Unmarshal(b, self)
 	}
 
 	return json.Unmarshal(b, v)

diff --git a/services/orgs.go b/services/orgs.go
@@ -71,7 +71,7 @@ type OrgManager interface {
 	CreateNewOrg(name, id string) (*api_proto.OrgRecord, error)
 	ListOrgs() []*api_proto.OrgRecord
 	GetOrg(org_id string) (*api_proto.OrgRecord, error)
-	DeleteOrg(ctx context.Context, org_id string) error
+	DeleteOrg(ctx context.Context, principal, org_id string) error
 
 	// The manager is responsible for running multiple services - one
 	// for each org. This ensures org services are separated out and

diff --git a/services/orgs/delete.go b/services/orgs/delete.go
@@ -16,17 +16,17 @@ import (
 )
 
 func RemoveOrgFromUsers(
-	ctx context.Context, org_id string) error {
+	ctx context.Context, principal, org_id string) error {
 
 	// Remove the org from all the users.
 	user_manager := services.GetUserManager()
-	users, err := user_manager.ListUsers(ctx)
+	users, err := user_manager.ListUsers(ctx, principal, services.LIST_ALL_ORGS)
 	if err != nil {
 		return err
 	}
 
 	for _, u := range users {
-		record, err := user_manager.GetUserWithHashes(ctx, u.Name)
+		record, err := user_manager.GetUserWithHashes(ctx, principal, u.Name)
 		if err == nil {
 			new_orgs := []*api_proto.OrgRecord{}
 			for _, org := range record.Orgs {
@@ -44,12 +44,12 @@ func RemoveOrgFromUsers(
 	return nil
 }
 
-func (self *OrgManager) DeleteOrg(ctx context.Context, org_id string) error {
+func (self *OrgManager) DeleteOrg(ctx context.Context, principal, org_id string) error {
 	if utils.IsRootOrg(org_id) {
 		return errors.New("Can not remove root org.")
 	}
 
-	err := RemoveOrgFromUsers(ctx, org_id)
+	err := RemoveOrgFromUsers(ctx, principal, org_id)
 	if err != nil {
 		return err
 	}