main.go 6.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245
  1. // Copyright 2022 gorse Project Authors
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package main
  15. import (
  16. "bufio"
  17. "bytes"
  18. "compress/gzip"
  19. "database/sql"
  20. _ "embed"
  21. "fmt"
  22. "io"
  23. "net/http"
  24. "os"
  25. "os/signal"
  26. "strings"
  27. "github.com/benhoyt/goawk/interp"
  28. "github.com/benhoyt/goawk/parser"
  29. "github.com/juju/errors"
  30. "github.com/schollz/progressbar/v3"
  31. "github.com/spf13/cobra"
  32. "github.com/zhenghaoz/gorse/base/log"
  33. "github.com/zhenghaoz/gorse/cmd/version"
  34. "github.com/zhenghaoz/gorse/config"
  35. "github.com/zhenghaoz/gorse/master"
  36. "github.com/zhenghaoz/gorse/storage"
  37. "github.com/zhenghaoz/gorse/storage/data"
  38. "github.com/zhenghaoz/gorse/worker"
  39. "go.uber.org/zap"
  40. )
  41. //go:embed mysql2sqlite
  42. var mysql2SQLite string
  43. const playgroundDataFile = "https://cdn.gorse.io/example/github.sql.gz"
  44. var oneCommand = &cobra.Command{
  45. Use: "gorse-in-one",
  46. Short: "The all in one distribution of gorse recommender system.",
  47. Run: func(cmd *cobra.Command, args []string) {
  48. // Show version
  49. if showVersion, _ := cmd.PersistentFlags().GetBool("version"); showVersion {
  50. fmt.Println(version.BuildInfo())
  51. return
  52. }
  53. // setup logger
  54. debug, _ := cmd.PersistentFlags().GetBool("debug")
  55. log.SetLogger(cmd.PersistentFlags(), debug)
  56. // load config
  57. var conf *config.Config
  58. var err error
  59. playgroundMode, _ := cmd.PersistentFlags().GetBool("playground")
  60. if playgroundMode {
  61. // load config for playground
  62. conf = config.GetDefaultConfig()
  63. conf.Database.DataStore = "sqlite://data.db"
  64. conf.Database.CacheStore = "sqlite://cache.db"
  65. conf.Recommend.DataSource.PositiveFeedbackTypes = []string{"star", "like"}
  66. conf.Recommend.DataSource.ReadFeedbackTypes = []string{"read"}
  67. if err := conf.Validate(true); err != nil {
  68. log.Logger().Fatal("invalid config", zap.Error(err))
  69. }
  70. fmt.Printf("Welcome to Gorse %s Playground\n", version.Version)
  71. if err = initializeDatabase("data.db"); err != nil {
  72. log.Logger().Fatal("failed to initialize database", zap.Error(err))
  73. }
  74. fmt.Println()
  75. fmt.Printf(" Dashboard: http://127.0.0.1:%d/overview\n", conf.Master.HttpPort)
  76. fmt.Printf(" RESTful APIs: http://127.0.0.1:%d/apidocs\n", conf.Master.HttpPort)
  77. fmt.Printf(" Documentation: https://gorse.io/docs\n")
  78. fmt.Println()
  79. } else {
  80. configPath, _ := cmd.PersistentFlags().GetString("config")
  81. log.Logger().Info("load config", zap.String("config", configPath))
  82. conf, err = config.LoadConfig(configPath, true)
  83. if err != nil {
  84. log.Logger().Fatal("failed to load config", zap.Error(err))
  85. }
  86. }
  87. // create master
  88. cachePath, _ := cmd.PersistentFlags().GetString("cache-path")
  89. managedMode, _ := cmd.PersistentFlags().GetBool("managed")
  90. m := master.NewMaster(conf, cachePath, managedMode)
  91. // Start worker
  92. workerJobs, _ := cmd.PersistentFlags().GetInt("recommend-jobs")
  93. w := worker.NewWorker(conf.Master.Host, conf.Master.Port, conf.Master.Host,
  94. 0, workerJobs, "", managedMode)
  95. go func() {
  96. w.SetOneMode(m.Settings)
  97. w.Serve()
  98. }()
  99. // Stop master
  100. done := make(chan struct{})
  101. go func() {
  102. sigint := make(chan os.Signal, 1)
  103. signal.Notify(sigint, os.Interrupt)
  104. <-sigint
  105. m.Shutdown()
  106. close(done)
  107. }()
  108. // Start master
  109. m.SetOneMode(w.ScheduleAPIHandler)
  110. m.Serve()
  111. <-done
  112. log.Logger().Info("stop gorse-in-one successfully")
  113. },
  114. }
  115. func init() {
  116. log.AddFlags(oneCommand.PersistentFlags())
  117. oneCommand.PersistentFlags().Bool("debug", false, "use debug log mode")
  118. oneCommand.PersistentFlags().Bool("managed", false, "enable managed mode")
  119. oneCommand.PersistentFlags().BoolP("version", "v", false, "gorse version")
  120. oneCommand.PersistentFlags().Bool("playground", false, "playground mode (setup a recommender system for GitHub repositories)")
  121. oneCommand.PersistentFlags().StringP("config", "c", "", "configuration file path")
  122. oneCommand.PersistentFlags().String("cache-path", "one_cache.data", "path of cache file")
  123. oneCommand.PersistentFlags().Int("recommend-jobs", 1, "number of working jobs for recommendation tasks")
  124. }
  125. func main() {
  126. if err := oneCommand.Execute(); err != nil {
  127. log.Logger().Fatal("failed to execute", zap.Error(err))
  128. }
  129. }
  130. func initializeDatabase(path string) error {
  131. // skip initialization if file exists
  132. if _, err := os.Stat(path); err == nil {
  133. return nil
  134. }
  135. // init database
  136. databaseClient, err := data.Open(storage.SQLitePrefix+path, "")
  137. if err != nil {
  138. return errors.Trace(err)
  139. }
  140. if err = databaseClient.Init(); err != nil {
  141. return errors.Trace(err)
  142. }
  143. if err = databaseClient.Close(); err != nil {
  144. return errors.Trace(err)
  145. }
  146. // open database
  147. db, err := sql.Open("sqlite", path)
  148. if err != nil {
  149. return errors.Trace(err)
  150. }
  151. defer db.Close()
  152. // download mysqldump file
  153. resp, err := http.Get(playgroundDataFile)
  154. if err != nil {
  155. return errors.Trace(err)
  156. }
  157. // create converter
  158. converter, err := NewMySQLToSQLiteConverter(mysql2SQLite)
  159. if err != nil {
  160. return errors.Trace(err)
  161. }
  162. // load mysqldump file
  163. pbReader := progressbar.NewReader(resp.Body, progressbar.DefaultBytes(
  164. resp.ContentLength,
  165. "Downloading playground dataset",
  166. ))
  167. gzipReader, err := gzip.NewReader(&pbReader)
  168. if err != nil {
  169. return errors.Trace(err)
  170. }
  171. reader := bufio.NewReader(gzipReader)
  172. var builder strings.Builder
  173. for {
  174. line, isPrefix, err := reader.ReadLine()
  175. if err == io.EOF {
  176. break
  177. } else if err != nil {
  178. return errors.Trace(err)
  179. }
  180. builder.Write(line)
  181. if !isPrefix {
  182. text := builder.String()
  183. if strings.HasPrefix(text, "INSERT INTO ") {
  184. // convert to SQLite sql
  185. sqliteSQL, err := converter.Convert(text)
  186. if err != nil {
  187. return errors.Trace(err)
  188. }
  189. if _, err = db.Exec(sqliteSQL); err != nil {
  190. return errors.Trace(err)
  191. }
  192. }
  193. builder.Reset()
  194. }
  195. }
  196. return nil
  197. }
  198. type MySQLToSQLiteConverter struct {
  199. interpreter *interp.Interpreter
  200. }
  201. func NewMySQLToSQLiteConverter(source string) (*MySQLToSQLiteConverter, error) {
  202. converter := &MySQLToSQLiteConverter{}
  203. program, err := parser.ParseProgram([]byte(source), nil)
  204. if err != nil {
  205. return nil, errors.Trace(err)
  206. }
  207. converter.interpreter, err = interp.New(program)
  208. if err != nil {
  209. return nil, errors.Trace(err)
  210. }
  211. return converter, nil
  212. }
  213. func (converter *MySQLToSQLiteConverter) Convert(sql string) (string, error) {
  214. input := strings.NewReader(sql)
  215. output := bytes.NewBuffer(nil)
  216. if _, err := converter.interpreter.Execute(&interp.Config{
  217. Stdin: input, Output: output, Args: []string{"-"},
  218. }); err != nil {
  219. return "", errors.Trace(err)
  220. }
  221. return output.String(), nil
  222. }