refactor(migrations): move migrations into a single package

ee/query-service/app/server.go

@@ -63,7 +63,6 @@ import (
 const AppDbEngine = "sqlite"
 
 type ServerOptions struct {
-	SigNoz            *signoz.SigNoz
 	PromConfigPath    string
 	SkipTopLvlOpsPath string
 	HTTPHostPort      string
@@ -81,7 +80,6 @@ type ServerOptions struct {
 	GatewayUrl        string
 	UseLogsNewSchema  bool
 	UseTraceNewSchema bool
-	Config            signoz.Config
 }
 
 // Server runs HTTP api service
@@ -111,34 +109,23 @@ func (s Server) HealthCheckStatus() chan healthcheck.Status {
 }
 
 // NewServer creates and initializes Server
-func NewServer(serverOptions *ServerOptions) (*Server, error) {
-
-	modelDao, err := dao.InitDao("sqlite", baseconst.RELATIONAL_DATASOURCE_PATH)
+func NewServer(serverOptions *ServerOptions, config signoz.Config, signoz *signoz.SigNoz) (*Server, error) {
+	modelDao, err := dao.InitDao(signoz.SqlStore.Provider().SqlxDB())
 	if err != nil {
 		return nil, err
 	}
 
-	baseexplorer.InitWithDSN(baseconst.RELATIONAL_DATASOURCE_PATH)
-
-	if err := preferences.InitDB(baseconst.RELATIONAL_DATASOURCE_PATH); err != nil {
-		return nil, err
-	}
-
-	localDB, err := dashboards.InitDB(baseconst.RELATIONAL_DATASOURCE_PATH)
-
-	if err != nil {
-		return nil, err
-	}
-
-	localDB.SetMaxOpenConns(10)
+	baseexplorer.InitWithDB(signoz.SqlStore.Provider().SqlxDB())
+	preferences.InitDB(signoz.SqlStore.Provider().SqlxDB())
+	dashboards.InitDB(signoz.SqlStore.Provider().SqlxDB())
 
 	gatewayProxy, err := gateway.NewProxy(serverOptions.GatewayUrl, gateway.RoutePrefix)
 	if err != nil {
 		return nil, err
 	}
 
 	// initiate license manager
-	lm, err := licensepkg.StartManager("sqlite", localDB)
+	lm, err := licensepkg.StartManager("sqlite", signoz.SqlStore.Provider().SqlxDB())
 	if err != nil {
 		return nil, err
 	}
@@ -152,7 +139,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 	if storage == "clickhouse" {
 		zap.L().Info("Using ClickHouse as datastore ...")
 		qb := db.NewDataConnector(
-			localDB,
+			signoz.SqlStore.Provider().SqlxDB(),
 			serverOptions.PromConfigPath,
 			lm,
 			serverOptions.MaxIdleConns,
@@ -188,7 +175,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 	rm, err := makeRulesManager(serverOptions.PromConfigPath,
 		baseconst.GetAlertManagerApiPrefix(),
 		serverOptions.RuleRepoURL,
-		localDB,
+		signoz.SqlStore.Provider().SqlxDB(),
 		reader,
 		c,
 		serverOptions.DisableRules,
@@ -209,12 +196,9 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 	}()
 
 	// initiate opamp
-	_, err = opAmpModel.InitDB(localDB)
-	if err != nil {
-		return nil, err
-	}
+	_ = opAmpModel.InitDB(signoz.SqlStore.Provider().SqlxDB())
 
-	integrationsController, err := integrations.NewController(localDB)
+	integrationsController, err := integrations.NewController(signoz.SqlStore.Provider().SqlxDB())
 	if err != nil {
 		return nil, fmt.Errorf(
 			"couldn't create integrations controller: %w", err,
@@ -223,15 +207,15 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 
 	// ingestion pipelines manager
 	logParsingPipelineController, err := logparsingpipeline.NewLogParsingPipelinesController(
-		localDB, "sqlite", integrationsController.GetPipelinesForInstalledIntegrations,
+		signoz.SqlStore.Provider().SqlxDB(), "sqlite", integrationsController.GetPipelinesForInstalledIntegrations,
 	)
 	if err != nil {
 		return nil, err
 	}
 
 	// initiate agent config handler
 	agentConfMgr, err := agentConf.Initiate(&agentConf.ManagerOptions{
-		DB:            localDB,
+		DB:            signoz.SqlStore.Provider().SqlxDB(),
 		DBEngine:      AppDbEngine,
 		AgentFeatures: []agentConf.AgentFeature{logParsingPipelineController},
 	})
@@ -293,7 +277,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 		usageManager:       usageManager,
 	}
 
-	httpServer, err := s.createPublicServer(apiHandler, serverOptions.SigNoz.Web)
+	httpServer, err := s.createPublicServer(apiHandler, signoz.Web)
 
 	if err != nil {
 		return nil, err

ee/query-service/dao/factory.go

@@ -1,18 +1,11 @@
 package dao
 
 import (
-	"fmt"
-
+	"github.com/jmoiron/sqlx"
 	"go.signoz.io/signoz/ee/query-service/dao/sqlite"
 )
 
-func InitDao(engine, path string) (ModelDao, error) {
-
-	switch engine {
-	case "sqlite":
-		return sqlite.InitDB(path)
-	default:
-		return nil, fmt.Errorf("qsdb type: %s is not supported in query service", engine)
-	}
+func InitDao(inputDB *sqlx.DB) (ModelDao, error) {
+	return sqlite.InitDB(inputDB)
 
 }

ee/query-service/dao/sqlite/modelDao.go

@@ -65,8 +65,8 @@ func columnExists(db *sqlx.DB, tableName, columnName string) bool {
 }
 
 // InitDB creates and extends base model DB repository
-func InitDB(dataSourceName string) (*modelDao, error) {
-	dao, err := basedsql.InitDB(dataSourceName)
+func InitDB(inputDB *sqlx.DB) (*modelDao, error) {
+	dao, err := basedsql.InitDB(inputDB)
 	if err != nil {
 		return nil, err
 	}

ee/query-service/main.go

@@ -18,7 +18,6 @@ import (
 	"go.signoz.io/signoz/pkg/config/provider/envprovider"
 	"go.signoz.io/signoz/pkg/query-service/auth"
 	baseconst "go.signoz.io/signoz/pkg/query-service/constants"
-	"go.signoz.io/signoz/pkg/query-service/migrate"
 	"go.signoz.io/signoz/pkg/query-service/version"
 	"go.signoz.io/signoz/pkg/signoz"
 	"google.golang.org/grpc"
@@ -150,7 +149,6 @@ func main() {
 	}
 
 	serverOptions := &app.ServerOptions{
-		SigNoz:            signoz,
 		HTTPHostPort:      baseconst.HTTPHostPort,
 		PromConfigPath:    promConfigPath,
 		SkipTopLvlOpsPath: skipTopLvlOpsPath,
@@ -167,7 +165,6 @@ func main() {
 		GatewayUrl:        gatewayUrl,
 		UseLogsNewSchema:  useLogsNewSchema,
 		UseTraceNewSchema: useTraceNewSchema,
-		Config:            config,
 	}
 
 	// Read the jwt secret key
@@ -179,13 +176,7 @@ func main() {
 		zap.L().Info("JWT secret key set successfully.")
 	}
 
-	if err := migrate.Migrate(baseconst.RELATIONAL_DATASOURCE_PATH); err != nil {
-		zap.L().Error("Failed to migrate", zap.Error(err))
-	} else {
-		zap.L().Info("Migration successful")
-	}
-
-	server, err := app.NewServer(serverOptions)
+	server, err := app.NewServer(serverOptions, config, signoz)
 	if err != nil {
 		zap.L().Fatal("Failed to create server", zap.Error(err))
 	}

go.mod

@@ -50,6 +50,9 @@ require (
 	github.com/soheilhy/cmux v0.1.5
 	github.com/srikanthccv/ClickHouse-go-mock v0.9.0
 	github.com/stretchr/testify v1.9.0
+	github.com/uptrace/bun v1.2.8
+	github.com/uptrace/bun/dialect/sqlitedialect v1.2.8
+	github.com/uptrace/bun/driver/sqliteshim v1.2.8
 	go.opentelemetry.io/collector/confmap v1.17.0
 	go.opentelemetry.io/collector/pdata v1.17.0
 	go.opentelemetry.io/collector/processor v0.111.0
@@ -63,7 +66,7 @@ require (
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.27.0
 	golang.org/x/crypto v0.31.0
-	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
+	golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
 	golang.org/x/net v0.29.0
 	golang.org/x/oauth2 v0.23.0
 	golang.org/x/text v0.21.0
@@ -121,7 +124,9 @@ require (
 	github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect
 	github.com/hashicorp/go-version v1.7.0 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jonboulle/clockwork v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -134,13 +139,15 @@ require (
 	github.com/lufia/plan9stats v0.0.0-20240408141607-282e7b5d6b74 // indirect
 	github.com/magefile/mage v1.15.0 // indirect
 	github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/oklog/run v1.1.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.111.0 // indirect
 	github.com/paulmach/orb v0.11.1 // indirect
@@ -152,6 +159,8 @@ require (
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common/sigv4 v0.1.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/puzpuzpuz/xsync/v3 v3.4.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/segmentio/backo-go v1.0.1 // indirect
@@ -163,8 +172,11 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tklauser/go-sysconf v0.3.13 // indirect
 	github.com/tklauser/numcpus v0.7.0 // indirect
+	github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
 	github.com/valyala/fastjson v1.6.4 // indirect
 	github.com/vjeantet/grok v1.0.1 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.opencensus.io v0.24.0 // indirect
@@ -213,7 +225,7 @@ require (
 	go.opentelemetry.io/otel/sdk/metric v1.30.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
-	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
 	golang.org/x/time v0.6.0 // indirect
 	gonum.org/v1/gonum v0.15.1 // indirect
 	google.golang.org/api v0.199.0 // indirect
@@ -222,6 +234,13 @@ require (
 	k8s.io/client-go v0.31.1 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
+	modernc.org/gc/v3 v3.0.0-20250105121824-520be1a3aee6 // indirect
+	modernc.org/libc v1.61.6 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.8.1 // indirect
+	modernc.org/sqlite v1.34.4 // indirect
+	modernc.org/strutil v1.2.1 // indirect
+	modernc.org/token v1.1.0 // indirect
 )
 
 replace github.com/prometheus/prometheus => github.com/SigNoz/prometheus v1.12.0