Added a gRPC based remote datastore (Velocidex#1366)

This allows minions to connect to the memcache data store of the master.

acls/acls.go

@@ -110,6 +110,9 @@ const (
 	// Allowed to create zip files.
 	PREPARE_RESULTS
 
+	// Allowed raw datastore access
+	DATASTORE_ACCESS
+
 	// When adding new permission - update CheckAccess,
 	// GetRolePermissions and acl.proto
 )
@@ -150,6 +153,8 @@ func (self ACL_PERMISSION) String() string {
 		return "MACHINE_STATE"
 	case PREPARE_RESULTS:
 		return "PREPARE_RESULTS"
+	case DATASTORE_ACCESS:
+		return "DATASTORE_ACCESS"
 
 	}
 	return fmt.Sprintf("%d", self)
@@ -192,6 +197,8 @@ func GetPermission(name string) ACL_PERMISSION {
 		return MACHINE_STATE
 	case "PREPARE_RESULTS":
 		return PREPARE_RESULTS
+	case "DATASTORE_ACCESS":
+		return DATASTORE_ACCESS
 
 	}
 	return NO_PERMISSIONS
@@ -345,6 +352,9 @@ func CheckAccessWithToken(
 	case PREPARE_RESULTS:
 		return token.PrepareResults, nil
 
+	case DATASTORE_ACCESS:
+		return token.DatastoreAccess, nil
+
 	}
 
 	return false, nil

acls/proto/acl.proto

@@ -33,6 +33,7 @@ message ApiClientACL {
     bool filesystem_write = 14;
     bool machine_state = 16;
     bool prepare_results = 17;
+    bool datastore_access = 18;
 
     // A list of roles in lieu of the permissions above. These will be
     // interpolated into this ACL object.

api/datastore.go

@@ -0,0 +1,132 @@
+package api
+
+import (
+	"github.com/golang/protobuf/ptypes/empty"
+	context "golang.org/x/net/context"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"www.velocidex.com/golang/velociraptor/acls"
+	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
+	"www.velocidex.com/golang/velociraptor/datastore"
+	"www.velocidex.com/golang/velociraptor/file_store/api"
+	"www.velocidex.com/golang/velociraptor/file_store/path_specs"
+)
+
+// Raw Datastore access requires the DATASTORE_ACCESS permission. This
+// is not provided by any role so the only callers allowed are
+// server-server gRPC calls (e.g. minion -> master)
+func (self *ApiServer) GetSubject(
+	ctx context.Context,
+	in *api_proto.DataRequest) (*api_proto.DataResponse, error) {
+
+	user_name := GetGRPCUserInfo(self.config, ctx, self.ca_pool).Name
+	perm, err := acls.CheckAccess(self.config, user_name, acls.DATASTORE_ACCESS)
+	if !perm || err != nil {
+		return nil, status.Error(codes.PermissionDenied,
+			"User is not allowed to access datastore.")
+	}
+
+	db, err := datastore.GetDB(self.config)
+	if err != nil {
+		return nil, err
+	}
+
+	raw_db, ok := db.(datastore.RawDataStore)
+	if !ok {
+		return nil, status.Error(codes.Internal,
+			"Datastore has no raw access.")
+	}
+
+	data, err := raw_db.GetBuffer(self.config, getURN(in))
+	return &api_proto.DataResponse{
+		Data: data,
+	}, err
+}
+
+func (self *ApiServer) SetSubject(
+	ctx context.Context,
+	in *api_proto.DataRequest) (*api_proto.DataResponse, error) {
+
+	user_name := GetGRPCUserInfo(self.config, ctx, self.ca_pool).Name
+	perm, err := acls.CheckAccess(self.config, user_name, acls.DATASTORE_ACCESS)
+	if !perm || err != nil {
+		return nil, status.Error(codes.PermissionDenied,
+			"User is not allowed to access datastore.")
+	}
+
+	db, err := datastore.GetDB(self.config)
+	if err != nil {
+		return nil, err
+	}
+
+	raw_db, ok := db.(datastore.RawDataStore)
+	if !ok {
+		return nil, status.Error(codes.Internal,
+			"Datastore has no raw access.")
+	}
+
+	err = raw_db.SetBuffer(self.config, getURN(in), in.Data)
+	return &api_proto.DataResponse{}, err
+}
+
+func (self *ApiServer) ListChildren(
+	ctx context.Context,
+	in *api_proto.DataRequest) (*api_proto.ListChildrenResponse, error) {
+
+	user_name := GetGRPCUserInfo(self.config, ctx, self.ca_pool).Name
+	perm, err := acls.CheckAccess(self.config, user_name, acls.DATASTORE_ACCESS)
+	if !perm || err != nil {
+		return nil, status.Error(codes.PermissionDenied,
+			"User is not allowed to access datastore.")
+	}
+
+	db, err := datastore.GetDB(self.config)
+	if err != nil {
+		return nil, err
+	}
+
+	children, err := db.ListChildren(self.config, getURN(in))
+	if err != nil {
+		return nil, err
+	}
+
+	result := &api_proto.ListChildrenResponse{}
+	for _, child := range children {
+		result.Children = append(result.Children, &api_proto.DSPathSpec{
+			Components: child.Components(),
+			PathType:   int64(child.Type()),
+			IsDir:      child.IsDir(),
+		})
+	}
+
+	return result, nil
+}
+
+func (self *ApiServer) DeleteSubject(
+	ctx context.Context,
+	in *api_proto.DataRequest) (*empty.Empty, error) {
+
+	user_name := GetGRPCUserInfo(self.config, ctx, self.ca_pool).Name
+	perm, err := acls.CheckAccess(self.config, user_name, acls.DATASTORE_ACCESS)
+	if !perm || err != nil {
+		return nil, status.Error(codes.PermissionDenied,
+			"User is not allowed to access datastore.")
+	}
+
+	db, err := datastore.GetDB(self.config)
+	if err != nil {
+		return nil, err
+	}
+
+	return &empty.Empty{}, db.DeleteSubject(self.config, getURN(in))
+}
+
+func getURN(in *api_proto.DataRequest) api.DSPathSpec {
+	path_spec := in.Pathspec
+	if path_spec == nil {
+		path_spec = &api_proto.DSPathSpec{}
+	}
+
+	return path_specs.NewUnsafeDatastorePath(
+		path_spec.Components...).SetType(api.PathType(path_spec.PathType))
+}

api/datastore_test.go

@@ -0,0 +1,132 @@
+package api
+
+import (
+	"testing"
+
+	"github.com/sebdah/goldie"
+	"github.com/stretchr/testify/suite"
+	"google.golang.org/protobuf/proto"
+	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
+	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+	"www.velocidex.com/golang/velociraptor/datastore"
+	"www.velocidex.com/golang/velociraptor/file_store/path_specs"
+	"www.velocidex.com/golang/velociraptor/file_store/test_utils"
+	"www.velocidex.com/golang/velociraptor/grpc_client"
+	"www.velocidex.com/golang/velociraptor/json"
+	"www.velocidex.com/golang/velociraptor/vtesting/assert"
+)
+
+type DatastoreAPITest struct {
+	test_utils.TestSuite
+
+	client_config *config_proto.Config
+}
+
+func (self *DatastoreAPITest) SetupTest() {
+	self.TestSuite.SetupTest()
+
+	// Now bring up an API server.
+	self.ConfigObj.Frontend.ServerServices = &config_proto.ServerServicesConfig{}
+	self.ConfigObj.API.BindPort = 8101
+
+	server_builder, err := NewServerBuilder(
+		self.Sm.Ctx, self.ConfigObj, self.Sm.Wg)
+	assert.NoError(self.T(), err)
+
+	err = server_builder.WithAPIServer(self.Sm.Ctx, self.Sm.Wg)
+	assert.NoError(self.T(), err)
+
+}
+
+func (self *DatastoreAPITest) TestDatastore() {
+	db, err := datastore.GetDB(self.ConfigObj)
+	assert.NoError(self.T(), err)
+
+	path_spec := path_specs.NewUnsafeDatastorePath("A", "B", "C")
+	sample := &api_proto.AgentInformation{Name: "Velociraptor"}
+	assert.NoError(self.T(),
+		db.SetSubject(self.ConfigObj, path_spec, sample))
+
+	// Make some RPC calls
+	conn, closer, err := grpc_client.Factory.GetAPIClient(
+		self.Sm.Ctx, self.ConfigObj)
+	assert.NoError(self.T(), err)
+	defer closer()
+
+	res, err := conn.GetSubject(self.Sm.Ctx, &api_proto.DataRequest{
+		Pathspec: &api_proto.DSPathSpec{
+			Components: path_spec.Components(),
+		}})
+	assert.NoError(self.T(), err)
+	assert.Equal(self.T(), res.Data, []byte("{\"name\":\"Velociraptor\"}"))
+
+	// Now set data through gRPC and read it using the standard
+	// datastore.
+	path_spec2 := path_specs.NewUnsafeDatastorePath("A", "B", "D")
+	_, err = conn.SetSubject(self.Sm.Ctx, &api_proto.DataRequest{
+		Data: []byte("{\"name\":\"Another Name\"}"),
+		Pathspec: &api_proto.DSPathSpec{
+			Components: path_spec2.Components(),
+		}})
+	assert.NoError(self.T(), err)
+
+	assert.NoError(self.T(),
+		db.GetSubject(self.ConfigObj, path_spec2, sample))
+	assert.Equal(self.T(), sample.Name, "Another Name")
+
+	// Now list the children
+	res2, err := conn.ListChildren(self.Sm.Ctx, &api_proto.DataRequest{
+		Pathspec: &api_proto.DSPathSpec{
+			Components: path_spec.Dir().Components(),
+		}})
+	assert.NoError(self.T(), err)
+	goldie.Assert(self.T(), "TestDatastore", json.MustMarshalIndent(res2))
+}
+
+func (self *DatastoreAPITest) TestRemoteDatastore() {
+	config_obj := proto.Clone(self.ConfigObj).(*config_proto.Config)
+	config_obj.Datastore.Implementation = "RemoteFileDataStore"
+
+	db, err := datastore.GetDB(config_obj)
+	assert.NoError(self.T(), err)
+
+	path_spec := path_specs.NewUnsafeDatastorePath("A", "B", "C")
+	sample := &api_proto.AgentInformation{Name: "Velociraptor"}
+	assert.NoError(self.T(),
+		db.SetSubject(config_obj, path_spec, sample))
+
+	sample2 := &api_proto.AgentInformation{}
+	assert.NoError(self.T(),
+		db.GetSubject(config_obj, path_spec, sample2))
+
+	assert.Equal(self.T(), sample, sample2)
+
+	// Test ListDirectory
+	path_spec2 := path_specs.NewUnsafeDatastorePath("A", "B", "D")
+	assert.NoError(self.T(),
+		db.SetSubject(config_obj, path_spec2, sample))
+
+	children, err := db.ListChildren(config_obj, path_spec.Dir())
+	assert.NoError(self.T(), err)
+	assert.Equal(self.T(), 2, len(children))
+	assert.Equal(self.T(), path_spec, children[0])
+	assert.Equal(self.T(), path_spec2, children[1])
+
+	// Now delete one
+	assert.NoError(self.T(),
+		db.DeleteSubject(config_obj, path_spec))
+
+	children, err = db.ListChildren(config_obj, path_spec.Dir())
+	assert.NoError(self.T(), err)
+	assert.Equal(self.T(), 1, len(children))
+	assert.Equal(self.T(), path_spec2, children[0])
+
+	children, err = db.ListChildren(config_obj, path_spec.Dir().Dir())
+	assert.NoError(self.T(), err)
+	assert.Equal(self.T(), 1, len(children))
+	assert.True(self.T(), children[0].IsDir())
+}
+
+func TestAPIDatastore(t *testing.T) {
+	suite.Run(t, &DatastoreAPITest{})
+}

api/fixtures/TestDatastore.golden

@@ -0,0 +1,18 @@
+{
+ "children": [
+  {
+   "components": [
+    "A",
+    "B",
+    "C"
+   ]
+  },
+  {
+   "components": [
+    "A",
+    "B",
+    "D"
+   ]
+  }
+ ]
+}